diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,6395 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 3821,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00026171159382360636,
+      "grad_norm": 2.427435874938965,
+      "learning_rate": 1.3054830287206268e-08,
+      "logits/chosen": -2.452890634536743,
+      "logits/rejected": -2.3576245307922363,
+      "logps/chosen": -290.49053955078125,
+      "logps/rejected": -374.69940185546875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0026171159382360636,
+      "grad_norm": 2.4065892696380615,
+      "learning_rate": 1.3054830287206266e-07,
+      "logits/chosen": -2.280916452407837,
+      "logits/rejected": -2.18080735206604,
+      "logps/chosen": -279.5721435546875,
+      "logps/rejected": -245.38124084472656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4236111044883728,
+      "rewards/chosen": 0.0002959521661978215,
+      "rewards/margins": 4.458064722712152e-05,
+      "rewards/rejected": 0.0002513715880922973,
+      "step": 10
+    },
+    {
+      "epoch": 0.005234231876472127,
+      "grad_norm": 2.543537139892578,
+      "learning_rate": 2.610966057441253e-07,
+      "logits/chosen": -2.286400318145752,
+      "logits/rejected": -2.1322734355926514,
+      "logps/chosen": -305.47900390625,
+      "logps/rejected": -237.6411895751953,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0033905524760484695,
+      "rewards/margins": 0.0010894734878093004,
+      "rewards/rejected": 0.002301078988239169,
+      "step": 20
+    },
+    {
+      "epoch": 0.007851347814708191,
+      "grad_norm": 2.317607879638672,
+      "learning_rate": 3.9164490861618804e-07,
+      "logits/chosen": -2.2721304893493652,
+      "logits/rejected": -2.2249627113342285,
+      "logps/chosen": -251.0873565673828,
+      "logps/rejected": -251.26864624023438,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.012264861725270748,
+      "rewards/margins": 0.0016630779718980193,
+      "rewards/rejected": 0.010601785033941269,
+      "step": 30
+    },
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 1.9544142484664917,
+      "learning_rate": 5.221932114882506e-07,
+      "logits/chosen": -2.1681597232818604,
+      "logits/rejected": -2.1325502395629883,
+      "logps/chosen": -216.1050262451172,
+      "logps/rejected": -221.6034698486328,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.019591109827160835,
+      "rewards/margins": 0.00413005193695426,
+      "rewards/rejected": 0.015461057424545288,
+      "step": 40
+    },
+    {
+      "epoch": 0.01308557969118032,
+      "grad_norm": 2.0900888442993164,
+      "learning_rate": 6.527415143603135e-07,
+      "logits/chosen": -2.2135119438171387,
+      "logits/rejected": -2.1745445728302,
+      "logps/chosen": -266.76007080078125,
+      "logps/rejected": -234.2284698486328,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02980896458029747,
+      "rewards/margins": 0.005023510195314884,
+      "rewards/rejected": 0.02478545531630516,
+      "step": 50
+    },
+    {
+      "epoch": 0.015702695629416383,
+      "grad_norm": 2.1390092372894287,
+      "learning_rate": 7.832898172323761e-07,
+      "logits/chosen": -2.1692872047424316,
+      "logits/rejected": -2.1056342124938965,
+      "logps/chosen": -252.186767578125,
+      "logps/rejected": -226.5349884033203,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.03267771750688553,
+      "rewards/margins": 0.0062465183436870575,
+      "rewards/rejected": 0.026431197300553322,
+      "step": 60
+    },
+    {
+      "epoch": 0.018319811567652448,
+      "grad_norm": 2.0599091053009033,
+      "learning_rate": 9.138381201044387e-07,
+      "logits/chosen": -2.309943675994873,
+      "logits/rejected": -2.187107563018799,
+      "logps/chosen": -271.86541748046875,
+      "logps/rejected": -246.50680541992188,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.04242750257253647,
+      "rewards/margins": 0.011090461164712906,
+      "rewards/rejected": 0.031337037682533264,
+      "step": 70
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 2.3880298137664795,
+      "learning_rate": 1.0443864229765013e-06,
+      "logits/chosen": -2.2041609287261963,
+      "logits/rejected": -2.1138315200805664,
+      "logps/chosen": -257.4185485839844,
+      "logps/rejected": -246.7639923095703,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03884550929069519,
+      "rewards/margins": 0.011877561919391155,
+      "rewards/rejected": 0.02696794643998146,
+      "step": 80
+    },
+    {
+      "epoch": 0.023554043444124574,
+      "grad_norm": 2.3031389713287354,
+      "learning_rate": 1.1749347258485642e-06,
+      "logits/chosen": -2.208482265472412,
+      "logits/rejected": -2.1343834400177,
+      "logps/chosen": -249.96255493164062,
+      "logps/rejected": -234.4242706298828,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.043103523552417755,
+      "rewards/margins": 0.018813790753483772,
+      "rewards/rejected": 0.024289730936288834,
+      "step": 90
+    },
+    {
+      "epoch": 0.02617115938236064,
+      "grad_norm": 2.119929075241089,
+      "learning_rate": 1.305483028720627e-06,
+      "logits/chosen": -2.2504467964172363,
+      "logits/rejected": -2.178734540939331,
+      "logps/chosen": -246.7833251953125,
+      "logps/rejected": -230.8575897216797,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.04890027642250061,
+      "rewards/margins": 0.025924110785126686,
+      "rewards/rejected": 0.022976163774728775,
+      "step": 100
+    },
+    {
+      "epoch": 0.02617115938236064,
+      "eval_logits/chosen": -2.1481568813323975,
+      "eval_logits/rejected": -2.055117607116699,
+      "eval_logps/chosen": -259.46044921875,
+      "eval_logps/rejected": -242.01309204101562,
+      "eval_loss": 0.6808694005012512,
+      "eval_rewards/accuracies": 0.6554999947547913,
+      "eval_rewards/chosen": 0.05141494795680046,
+      "eval_rewards/margins": 0.025842413306236267,
+      "eval_rewards/rejected": 0.025572534650564194,
+      "eval_runtime": 1599.8543,
+      "eval_samples_per_second": 1.25,
+      "eval_steps_per_second": 0.156,
+      "step": 100
+    },
+    {
+      "epoch": 0.028788275320596704,
+      "grad_norm": 2.4198131561279297,
+      "learning_rate": 1.4360313315926894e-06,
+      "logits/chosen": -2.2423720359802246,
+      "logits/rejected": -2.1254634857177734,
+      "logps/chosen": -284.2754821777344,
+      "logps/rejected": -239.1751251220703,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.050642453134059906,
+      "rewards/margins": 0.03400001674890518,
+      "rewards/rejected": 0.016642430797219276,
+      "step": 110
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 2.272566556930542,
+      "learning_rate": 1.5665796344647521e-06,
+      "logits/chosen": -2.273714303970337,
+      "logits/rejected": -2.160338878631592,
+      "logps/chosen": -287.36285400390625,
+      "logps/rejected": -272.5426025390625,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.05583573505282402,
+      "rewards/margins": 0.04974224418401718,
+      "rewards/rejected": 0.006093493662774563,
+      "step": 120
+    },
+    {
+      "epoch": 0.03402250719706883,
+      "grad_norm": 2.827535390853882,
+      "learning_rate": 1.6971279373368146e-06,
+      "logits/chosen": -2.2895429134368896,
+      "logits/rejected": -2.1921463012695312,
+      "logps/chosen": -250.36807250976562,
+      "logps/rejected": -254.2834930419922,
+      "loss": 0.664,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.04658503085374832,
+      "rewards/margins": 0.06275991350412369,
+      "rewards/rejected": -0.01617487706243992,
+      "step": 130
+    },
+    {
+      "epoch": 0.036639623135304895,
+      "grad_norm": 2.8360142707824707,
+      "learning_rate": 1.8276762402088774e-06,
+      "logits/chosen": -2.285165309906006,
+      "logits/rejected": -2.075157880783081,
+      "logps/chosen": -272.5437927246094,
+      "logps/rejected": -229.80880737304688,
+      "loss": 0.662,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.021640608087182045,
+      "rewards/margins": 0.06897087395191193,
+      "rewards/rejected": -0.047330256551504135,
+      "step": 140
+    },
+    {
+      "epoch": 0.03925673907354096,
+      "grad_norm": 3.0254032611846924,
+      "learning_rate": 1.9582245430809403e-06,
+      "logits/chosen": -2.289304494857788,
+      "logits/rejected": -2.16825532913208,
+      "logps/chosen": -283.7846984863281,
+      "logps/rejected": -248.1438446044922,
+      "loss": 0.6606,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.0013313032686710358,
+      "rewards/margins": 0.07430683076381683,
+      "rewards/rejected": -0.0729755312204361,
+      "step": 150
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 3.2089273929595947,
+      "learning_rate": 2.0887728459530026e-06,
+      "logits/chosen": -2.209859609603882,
+      "logits/rejected": -2.1506431102752686,
+      "logps/chosen": -262.52569580078125,
+      "logps/rejected": -270.04766845703125,
+      "loss": 0.6632,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.05757613852620125,
+      "rewards/margins": 0.07334659993648529,
+      "rewards/rejected": -0.13092274963855743,
+      "step": 160
+    },
+    {
+      "epoch": 0.04449097095001309,
+      "grad_norm": 3.7007648944854736,
+      "learning_rate": 2.2193211488250653e-06,
+      "logits/chosen": -2.190873384475708,
+      "logits/rejected": -2.1105270385742188,
+      "logps/chosen": -227.3632049560547,
+      "logps/rejected": -236.8821563720703,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0691681057214737,
+      "rewards/margins": 0.0719941109418869,
+      "rewards/rejected": -0.1411622166633606,
+      "step": 170
+    },
+    {
+      "epoch": 0.04710808688824915,
+      "grad_norm": 5.2089338302612305,
+      "learning_rate": 2.3498694516971284e-06,
+      "logits/chosen": -2.2085936069488525,
+      "logits/rejected": -2.1274473667144775,
+      "logps/chosen": -273.2780456542969,
+      "logps/rejected": -269.22747802734375,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17684438824653625,
+      "rewards/margins": 0.0919983834028244,
+      "rewards/rejected": -0.26884278655052185,
+      "step": 180
+    },
+    {
+      "epoch": 0.04972520282648522,
+      "grad_norm": 4.031327724456787,
+      "learning_rate": 2.4804177545691907e-06,
+      "logits/chosen": -2.2836358547210693,
+      "logits/rejected": -2.1660006046295166,
+      "logps/chosen": -281.2835388183594,
+      "logps/rejected": -266.60821533203125,
+      "loss": 0.637,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.11275825649499893,
+      "rewards/margins": 0.13688938319683075,
+      "rewards/rejected": -0.24964764714241028,
+      "step": 190
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 6.425544261932373,
+      "learning_rate": 2.610966057441254e-06,
+      "logits/chosen": -2.2068774700164795,
+      "logits/rejected": -2.0849859714508057,
+      "logps/chosen": -266.81500244140625,
+      "logps/rejected": -241.287353515625,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.1718008667230606,
+      "rewards/margins": 0.12619325518608093,
+      "rewards/rejected": -0.29799407720565796,
+      "step": 200
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "eval_logits/chosen": -2.0999979972839355,
+      "eval_logits/rejected": -2.011294364929199,
+      "eval_logps/chosen": -283.4154357910156,
+      "eval_logps/rejected": -278.46148681640625,
+      "eval_loss": 0.6356053948402405,
+      "eval_rewards/accuracies": 0.6759999990463257,
+      "eval_rewards/chosen": -0.18813487887382507,
+      "eval_rewards/margins": 0.15077635645866394,
+      "eval_rewards/rejected": -0.3389112055301666,
+      "eval_runtime": 1598.5625,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 200
+    },
+    {
+      "epoch": 0.05495943470295734,
+      "grad_norm": 4.150570869445801,
+      "learning_rate": 2.741514360313316e-06,
+      "logits/chosen": -2.255913734436035,
+      "logits/rejected": -2.120842456817627,
+      "logps/chosen": -280.205810546875,
+      "logps/rejected": -268.5466613769531,
+      "loss": 0.6131,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18164019286632538,
+      "rewards/margins": 0.19993841648101807,
+      "rewards/rejected": -0.38157862424850464,
+      "step": 210
+    },
+    {
+      "epoch": 0.05757655064119341,
+      "grad_norm": 4.034811496734619,
+      "learning_rate": 2.872062663185379e-06,
+      "logits/chosen": -2.1413655281066895,
+      "logits/rejected": -2.093209743499756,
+      "logps/chosen": -298.56549072265625,
+      "logps/rejected": -289.4757995605469,
+      "loss": 0.6254,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.4465310573577881,
+      "rewards/margins": 0.17338070273399353,
+      "rewards/rejected": -0.619911789894104,
+      "step": 220
+    },
+    {
+      "epoch": 0.06019366657942947,
+      "grad_norm": 6.063634395599365,
+      "learning_rate": 3.0026109660574416e-06,
+      "logits/chosen": -2.266742706298828,
+      "logits/rejected": -2.164170742034912,
+      "logps/chosen": -352.3594665527344,
+      "logps/rejected": -333.99053955078125,
+      "loss": 0.6479,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.46258726716041565,
+      "rewards/margins": 0.17155149579048157,
+      "rewards/rejected": -0.6341387033462524,
+      "step": 230
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 5.352989673614502,
+      "learning_rate": 3.1331592689295043e-06,
+      "logits/chosen": -2.136970281600952,
+      "logits/rejected": -2.0645029544830322,
+      "logps/chosen": -342.98870849609375,
+      "logps/rejected": -350.2208251953125,
+      "loss": 0.6237,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.4678193926811218,
+      "rewards/margins": 0.2228115350008011,
+      "rewards/rejected": -0.6906309127807617,
+      "step": 240
+    },
+    {
+      "epoch": 0.06542789845590159,
+      "grad_norm": 6.087672233581543,
+      "learning_rate": 3.263707571801567e-06,
+      "logits/chosen": -2.162543296813965,
+      "logits/rejected": -2.1394383907318115,
+      "logps/chosen": -298.310302734375,
+      "logps/rejected": -299.23260498046875,
+      "loss": 0.6014,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3882629871368408,
+      "rewards/margins": 0.26374003291130066,
+      "rewards/rejected": -0.6520029902458191,
+      "step": 250
+    },
+    {
+      "epoch": 0.06804501439413765,
+      "grad_norm": 9.075284004211426,
+      "learning_rate": 3.3942558746736293e-06,
+      "logits/chosen": -2.2069649696350098,
+      "logits/rejected": -2.0832362174987793,
+      "logps/chosen": -315.36358642578125,
+      "logps/rejected": -315.8193664550781,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.43095770478248596,
+      "rewards/margins": 0.2242399901151657,
+      "rewards/rejected": -0.6551976203918457,
+      "step": 260
+    },
+    {
+      "epoch": 0.07066213033237373,
+      "grad_norm": 4.0352067947387695,
+      "learning_rate": 3.524804177545692e-06,
+      "logits/chosen": -2.1507174968719482,
+      "logits/rejected": -2.084745407104492,
+      "logps/chosen": -323.89361572265625,
+      "logps/rejected": -321.73907470703125,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6326006650924683,
+      "rewards/margins": 0.293195515871048,
+      "rewards/rejected": -0.9257962107658386,
+      "step": 270
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 6.033257007598877,
+      "learning_rate": 3.6553524804177547e-06,
+      "logits/chosen": -2.17421293258667,
+      "logits/rejected": -2.054452896118164,
+      "logps/chosen": -340.52557373046875,
+      "logps/rejected": -340.84259033203125,
+      "loss": 0.627,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8170977830886841,
+      "rewards/margins": 0.2173783779144287,
+      "rewards/rejected": -1.0344761610031128,
+      "step": 280
+    },
+    {
+      "epoch": 0.07589636220884585,
+      "grad_norm": 6.212845325469971,
+      "learning_rate": 3.7859007832898174e-06,
+      "logits/chosen": -2.1532936096191406,
+      "logits/rejected": -2.0849764347076416,
+      "logps/chosen": -353.55975341796875,
+      "logps/rejected": -354.45782470703125,
+      "loss": 0.6139,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6797887086868286,
+      "rewards/margins": 0.26041343808174133,
+      "rewards/rejected": -0.9402019381523132,
+      "step": 290
+    },
+    {
+      "epoch": 0.07851347814708191,
+      "grad_norm": 9.043365478515625,
+      "learning_rate": 3.9164490861618806e-06,
+      "logits/chosen": -2.1526737213134766,
+      "logits/rejected": -2.0208210945129395,
+      "logps/chosen": -306.4209899902344,
+      "logps/rejected": -320.6431884765625,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6231580972671509,
+      "rewards/margins": 0.2800864577293396,
+      "rewards/rejected": -0.9032446146011353,
+      "step": 300
+    },
+    {
+      "epoch": 0.07851347814708191,
+      "eval_logits/chosen": -2.0781641006469727,
+      "eval_logits/rejected": -1.9948630332946777,
+      "eval_logps/chosen": -333.25830078125,
+      "eval_logps/rejected": -342.0091247558594,
+      "eval_loss": 0.6053693890571594,
+      "eval_rewards/accuracies": 0.6815000176429749,
+      "eval_rewards/chosen": -0.6865635514259338,
+      "eval_rewards/margins": 0.2878238558769226,
+      "eval_rewards/rejected": -0.9743873476982117,
+      "eval_runtime": 1598.3515,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 300
+    },
+    {
+      "epoch": 0.08113059408531798,
+      "grad_norm": 8.154093742370605,
+      "learning_rate": 4.046997389033943e-06,
+      "logits/chosen": -2.2768380641937256,
+      "logits/rejected": -2.16288161277771,
+      "logps/chosen": -355.626953125,
+      "logps/rejected": -341.3152770996094,
+      "loss": 0.5637,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6677303910255432,
+      "rewards/margins": 0.39315730333328247,
+      "rewards/rejected": -1.0608876943588257,
+      "step": 310
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 12.870465278625488,
+      "learning_rate": 4.177545691906005e-06,
+      "logits/chosen": -2.2009758949279785,
+      "logits/rejected": -2.101364850997925,
+      "logps/chosen": -330.45904541015625,
+      "logps/rejected": -343.94305419921875,
+      "loss": 0.6039,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.7225368022918701,
+      "rewards/margins": 0.30548617243766785,
+      "rewards/rejected": -1.0280230045318604,
+      "step": 320
+    },
+    {
+      "epoch": 0.08636482596179011,
+      "grad_norm": 5.590673923492432,
+      "learning_rate": 4.308093994778068e-06,
+      "logits/chosen": -2.086026668548584,
+      "logits/rejected": -2.0480690002441406,
+      "logps/chosen": -313.5097351074219,
+      "logps/rejected": -321.5640869140625,
+      "loss": 0.5895,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5396815538406372,
+      "rewards/margins": 0.322670042514801,
+      "rewards/rejected": -0.8623515963554382,
+      "step": 330
+    },
+    {
+      "epoch": 0.08898194190002617,
+      "grad_norm": 7.664637088775635,
+      "learning_rate": 4.4386422976501306e-06,
+      "logits/chosen": -2.1061110496520996,
+      "logits/rejected": -2.049543857574463,
+      "logps/chosen": -358.71868896484375,
+      "logps/rejected": -383.11328125,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6914780735969543,
+      "rewards/margins": 0.45771294832229614,
+      "rewards/rejected": -1.1491910219192505,
+      "step": 340
+    },
+    {
+      "epoch": 0.09159905783826224,
+      "grad_norm": 10.34107780456543,
+      "learning_rate": 4.569190600522193e-06,
+      "logits/chosen": -1.9783916473388672,
+      "logits/rejected": -1.901391625404358,
+      "logps/chosen": -401.55120849609375,
+      "logps/rejected": -426.84832763671875,
+      "loss": 0.6179,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1243749856948853,
+      "rewards/margins": 0.3678717613220215,
+      "rewards/rejected": -1.4922468662261963,
+      "step": 350
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 6.533565044403076,
+      "learning_rate": 4.699738903394257e-06,
+      "logits/chosen": -1.9793760776519775,
+      "logits/rejected": -1.922286033630371,
+      "logps/chosen": -367.68817138671875,
+      "logps/rejected": -385.93798828125,
+      "loss": 0.5861,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.2674810886383057,
+      "rewards/margins": 0.39134687185287476,
+      "rewards/rejected": -1.6588280200958252,
+      "step": 360
+    },
+    {
+      "epoch": 0.09683328971473436,
+      "grad_norm": 9.993318557739258,
+      "learning_rate": 4.8302872062663196e-06,
+      "logits/chosen": -1.9993594884872437,
+      "logits/rejected": -1.8775148391723633,
+      "logps/chosen": -397.10125732421875,
+      "logps/rejected": -394.6053771972656,
+      "loss": 0.5852,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.2026114463806152,
+      "rewards/margins": 0.4132401943206787,
+      "rewards/rejected": -1.615851640701294,
+      "step": 370
+    },
+    {
+      "epoch": 0.09945040565297043,
+      "grad_norm": 8.581938743591309,
+      "learning_rate": 4.9608355091383814e-06,
+      "logits/chosen": -1.9525811672210693,
+      "logits/rejected": -1.7932708263397217,
+      "logps/chosen": -388.2157287597656,
+      "logps/rejected": -399.0534973144531,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9025002717971802,
+      "rewards/margins": 0.44248518347740173,
+      "rewards/rejected": -1.3449854850769043,
+      "step": 380
+    },
+    {
+      "epoch": 0.1020675215912065,
+      "grad_norm": 17.113487243652344,
+      "learning_rate": 4.9999488562447675e-06,
+      "logits/chosen": -1.920275330543518,
+      "logits/rejected": -1.8368419408798218,
+      "logps/chosen": -342.7761535644531,
+      "logps/rejected": -365.7699279785156,
+      "loss": 0.5702,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6181488633155823,
+      "rewards/margins": 0.4308691620826721,
+      "rewards/rejected": -1.0490180253982544,
+      "step": 390
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 18.049278259277344,
+      "learning_rate": 4.999698361256577e-06,
+      "logits/chosen": -1.844530463218689,
+      "logits/rejected": -1.724854826927185,
+      "logps/chosen": -372.54058837890625,
+      "logps/rejected": -370.39080810546875,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0718820095062256,
+      "rewards/margins": 0.3849504590034485,
+      "rewards/rejected": -1.4568325281143188,
+      "step": 400
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "eval_logits/chosen": -1.6757981777191162,
+      "eval_logits/rejected": -1.5843830108642578,
+      "eval_logps/chosen": -409.4522399902344,
+      "eval_logps/rejected": -440.5653381347656,
+      "eval_loss": 0.5824012160301208,
+      "eval_rewards/accuracies": 0.6830000281333923,
+      "eval_rewards/chosen": -1.448502540588379,
+      "eval_rewards/margins": 0.5114473700523376,
+      "eval_rewards/rejected": -1.9599499702453613,
+      "eval_runtime": 1596.5137,
+      "eval_samples_per_second": 1.253,
+      "eval_steps_per_second": 0.157,
+      "step": 400
+    },
+    {
+      "epoch": 0.10730175346767862,
+      "grad_norm": 7.473143577575684,
+      "learning_rate": 4.999239142174581e-06,
+      "logits/chosen": -1.7772619724273682,
+      "logits/rejected": -1.71932053565979,
+      "logps/chosen": -362.2901306152344,
+      "logps/rejected": -392.97271728515625,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2004709243774414,
+      "rewards/margins": 0.3244817852973938,
+      "rewards/rejected": -1.5249526500701904,
+      "step": 410
+    },
+    {
+      "epoch": 0.10991886940591468,
+      "grad_norm": 5.672206401824951,
+      "learning_rate": 4.99857123734344e-06,
+      "logits/chosen": -1.798180341720581,
+      "logits/rejected": -1.6758928298950195,
+      "logps/chosen": -342.5751037597656,
+      "logps/rejected": -385.69757080078125,
+      "loss": 0.5303,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1391090154647827,
+      "rewards/margins": 0.5212605595588684,
+      "rewards/rejected": -1.660369634628296,
+      "step": 420
+    },
+    {
+      "epoch": 0.11253598534415074,
+      "grad_norm": 7.920849800109863,
+      "learning_rate": 4.997694702533016e-06,
+      "logits/chosen": -1.839582085609436,
+      "logits/rejected": -1.7662830352783203,
+      "logps/chosen": -396.0287170410156,
+      "logps/rejected": -421.93707275390625,
+      "loss": 0.542,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1852526664733887,
+      "rewards/margins": 0.49562257528305054,
+      "rewards/rejected": -1.6808754205703735,
+      "step": 430
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 9.021227836608887,
+      "learning_rate": 4.996609610933713e-06,
+      "logits/chosen": -1.9095804691314697,
+      "logits/rejected": -1.8493105173110962,
+      "logps/chosen": -372.8834533691406,
+      "logps/rejected": -393.52532958984375,
+      "loss": 0.575,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0081276893615723,
+      "rewards/margins": 0.4901418089866638,
+      "rewards/rejected": -1.4982694387435913,
+      "step": 440
+    },
+    {
+      "epoch": 0.11777021722062288,
+      "grad_norm": 8.462127685546875,
+      "learning_rate": 4.995316053150366e-06,
+      "logits/chosen": -1.7473382949829102,
+      "logits/rejected": -1.6900889873504639,
+      "logps/chosen": -388.23175048828125,
+      "logps/rejected": -426.498046875,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2048237323760986,
+      "rewards/margins": 0.612975537776947,
+      "rewards/rejected": -1.8177993297576904,
+      "step": 450
+    },
+    {
+      "epoch": 0.12038733315885894,
+      "grad_norm": 12.204924583435059,
+      "learning_rate": 4.9938141371946815e-06,
+      "logits/chosen": -1.7278436422348022,
+      "logits/rejected": -1.6576976776123047,
+      "logps/chosen": -472.71600341796875,
+      "logps/rejected": -530.8753662109375,
+      "loss": 0.5586,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.0762267112731934,
+      "rewards/margins": 0.7278280258178711,
+      "rewards/rejected": -2.8040547370910645,
+      "step": 460
+    },
+    {
+      "epoch": 0.123004449097095,
+      "grad_norm": 6.473967552185059,
+      "learning_rate": 4.992103988476206e-06,
+      "logits/chosen": -1.8171007633209229,
+      "logits/rejected": -1.712386131286621,
+      "logps/chosen": -393.3753662109375,
+      "logps/rejected": -441.9925231933594,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4961185455322266,
+      "rewards/margins": 0.6115677356719971,
+      "rewards/rejected": -2.1076862812042236,
+      "step": 470
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 5.5269598960876465,
+      "learning_rate": 4.990185749791866e-06,
+      "logits/chosen": -1.9438987970352173,
+      "logits/rejected": -1.8433564901351929,
+      "logps/chosen": -341.74261474609375,
+      "logps/rejected": -403.5177917480469,
+      "loss": 0.5323,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8496583104133606,
+      "rewards/margins": 0.5994860529899597,
+      "rewards/rejected": -1.4491443634033203,
+      "step": 480
+    },
+    {
+      "epoch": 0.12823868097356714,
+      "grad_norm": 9.541817665100098,
+      "learning_rate": 4.9880595813140395e-06,
+      "logits/chosen": -1.9532935619354248,
+      "logits/rejected": -1.8457939624786377,
+      "logps/chosen": -374.6437072753906,
+      "logps/rejected": -403.85614013671875,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8984916806221008,
+      "rewards/margins": 0.5953295826911926,
+      "rewards/rejected": -1.493821144104004,
+      "step": 490
+    },
+    {
+      "epoch": 0.13085579691180318,
+      "grad_norm": 13.012099266052246,
+      "learning_rate": 4.985725660577184e-06,
+      "logits/chosen": -1.8454961776733398,
+      "logits/rejected": -1.7134668827056885,
+      "logps/chosen": -394.9266052246094,
+      "logps/rejected": -415.48138427734375,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.2125742435455322,
+      "rewards/margins": 0.6179044842720032,
+      "rewards/rejected": -1.8304786682128906,
+      "step": 500
+    },
+    {
+      "epoch": 0.13085579691180318,
+      "eval_logits/chosen": -1.6658297777175903,
+      "eval_logits/rejected": -1.56244695186615,
+      "eval_logps/chosen": -379.18035888671875,
+      "eval_logps/rejected": -420.46356201171875,
+      "eval_loss": 0.5725830793380737,
+      "eval_rewards/accuracies": 0.6915000081062317,
+      "eval_rewards/chosen": -1.1457839012145996,
+      "eval_rewards/margins": 0.6131481528282166,
+      "eval_rewards/rejected": -1.758932113647461,
+      "eval_runtime": 1597.6305,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 500
+    },
+    {
+      "epoch": 0.13347291285003926,
+      "grad_norm": 9.327789306640625,
+      "learning_rate": 4.983184182463009e-06,
+      "logits/chosen": -1.7511212825775146,
+      "logits/rejected": -1.6526283025741577,
+      "logps/chosen": -403.2107238769531,
+      "logps/rejected": -449.2831115722656,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.2511955499649048,
+      "rewards/margins": 0.830196738243103,
+      "rewards/rejected": -2.0813920497894287,
+      "step": 510
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 20.807098388671875,
+      "learning_rate": 4.980435359184203e-06,
+      "logits/chosen": -1.7725025415420532,
+      "logits/rejected": -1.7354393005371094,
+      "logps/chosen": -416.3721618652344,
+      "logps/rejected": -457.4166564941406,
+      "loss": 0.5914,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4335078001022339,
+      "rewards/margins": 0.5734153985977173,
+      "rewards/rejected": -2.006923198699951,
+      "step": 520
+    },
+    {
+      "epoch": 0.13870714472651138,
+      "grad_norm": 9.319575309753418,
+      "learning_rate": 4.9774794202667236e-06,
+      "logits/chosen": -1.8060506582260132,
+      "logits/rejected": -1.7954254150390625,
+      "logps/chosen": -347.8722229003906,
+      "logps/rejected": -408.69842529296875,
+      "loss": 0.568,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8692201375961304,
+      "rewards/margins": 0.5232836008071899,
+      "rewards/rejected": -1.3925037384033203,
+      "step": 530
+    },
+    {
+      "epoch": 0.14132426066474746,
+      "grad_norm": 18.703466415405273,
+      "learning_rate": 4.974316612530615e-06,
+      "logits/chosen": -1.718746542930603,
+      "logits/rejected": -1.6051177978515625,
+      "logps/chosen": -397.3692932128906,
+      "logps/rejected": -437.828125,
+      "loss": 0.4615,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1902121305465698,
+      "rewards/margins": 0.8715072870254517,
+      "rewards/rejected": -2.0617194175720215,
+      "step": 540
+    },
+    {
+      "epoch": 0.1439413766029835,
+      "grad_norm": 13.324700355529785,
+      "learning_rate": 4.970947200069416e-06,
+      "logits/chosen": -1.5196049213409424,
+      "logits/rejected": -1.463122010231018,
+      "logps/chosen": -523.3265380859375,
+      "logps/rejected": -561.0288696289062,
+      "loss": 0.6068,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.449286937713623,
+      "rewards/margins": 0.5750513076782227,
+      "rewards/rejected": -3.0243382453918457,
+      "step": 550
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 7.8647990226745605,
+      "learning_rate": 4.967371464228096e-06,
+      "logits/chosen": -1.6284644603729248,
+      "logits/rejected": -1.54505455493927,
+      "logps/chosen": -521.0708618164062,
+      "logps/rejected": -584.5469970703125,
+      "loss": 0.5398,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.636112689971924,
+      "rewards/margins": 0.6315088868141174,
+      "rewards/rejected": -3.2676215171813965,
+      "step": 560
+    },
+    {
+      "epoch": 0.14917560847945563,
+      "grad_norm": 8.891359329223633,
+      "learning_rate": 4.963589703579569e-06,
+      "logits/chosen": -1.7360155582427979,
+      "logits/rejected": -1.6161645650863647,
+      "logps/chosen": -575.2686767578125,
+      "logps/rejected": -591.63671875,
+      "loss": 0.6015,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.752664566040039,
+      "rewards/margins": 0.5114163160324097,
+      "rewards/rejected": -3.264080762863159,
+      "step": 570
+    },
+    {
+      "epoch": 0.1517927244176917,
+      "grad_norm": 13.137563705444336,
+      "learning_rate": 4.9596022338997615e-06,
+      "logits/chosen": -1.8721704483032227,
+      "logits/rejected": -1.7023900747299194,
+      "logps/chosen": -481.35992431640625,
+      "logps/rejected": -496.80792236328125,
+      "loss": 0.5479,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8363927602767944,
+      "rewards/margins": 0.565157949924469,
+      "rewards/rejected": -2.401550769805908,
+      "step": 580
+    },
+    {
+      "epoch": 0.15440984035592778,
+      "grad_norm": 9.32674789428711,
+      "learning_rate": 4.955409388141243e-06,
+      "logits/chosen": -1.9033949375152588,
+      "logits/rejected": -1.81440007686615,
+      "logps/chosen": -351.5168762207031,
+      "logps/rejected": -374.45159912109375,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9435514211654663,
+      "rewards/margins": 0.46548739075660706,
+      "rewards/rejected": -1.4090386629104614,
+      "step": 590
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 8.5282621383667,
+      "learning_rate": 4.951011516405429e-06,
+      "logits/chosen": -1.9967750310897827,
+      "logits/rejected": -1.959011435508728,
+      "logps/chosen": -324.1109924316406,
+      "logps/rejected": -367.7194519042969,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7343847155570984,
+      "rewards/margins": 0.5855700373649597,
+      "rewards/rejected": -1.319954752922058,
+      "step": 600
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "eval_logits/chosen": -1.7954951524734497,
+      "eval_logits/rejected": -1.6945267915725708,
+      "eval_logps/chosen": -377.46051025390625,
+      "eval_logps/rejected": -426.21209716796875,
+      "eval_loss": 0.5631101727485657,
+      "eval_rewards/accuracies": 0.703000009059906,
+      "eval_rewards/chosen": -1.1285854578018188,
+      "eval_rewards/margins": 0.6878318190574646,
+      "eval_rewards/rejected": -1.8164173364639282,
+      "eval_runtime": 1597.4817,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 600
+    },
+    {
+      "epoch": 0.1596440722323999,
+      "grad_norm": 8.468684196472168,
+      "learning_rate": 4.946408985913344e-06,
+      "logits/chosen": -1.8055137395858765,
+      "logits/rejected": -1.7273778915405273,
+      "logps/chosen": -379.7837829589844,
+      "logps/rejected": -452.4205627441406,
+      "loss": 0.5374,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.3405410051345825,
+      "rewards/margins": 0.9033535122871399,
+      "rewards/rejected": -2.243894577026367,
+      "step": 610
+    },
+    {
+      "epoch": 0.16226118817063595,
+      "grad_norm": 8.468870162963867,
+      "learning_rate": 4.941602180974958e-06,
+      "logits/chosen": -1.712264060974121,
+      "logits/rejected": -1.5265555381774902,
+      "logps/chosen": -442.0708923339844,
+      "logps/rejected": -476.46112060546875,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5436513423919678,
+      "rewards/margins": 0.9296489953994751,
+      "rewards/rejected": -2.4733004570007324,
+      "step": 620
+    },
+    {
+      "epoch": 0.16487830410887203,
+      "grad_norm": 8.517386436462402,
+      "learning_rate": 4.936591502957101e-06,
+      "logits/chosen": -1.650029182434082,
+      "logits/rejected": -1.5338895320892334,
+      "logps/chosen": -383.0647888183594,
+      "logps/rejected": -462.663330078125,
+      "loss": 0.5098,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3505860567092896,
+      "rewards/margins": 0.8982070684432983,
+      "rewards/rejected": -2.248793125152588,
+      "step": 630
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 7.095081329345703,
+      "learning_rate": 4.931377370249946e-06,
+      "logits/chosen": -1.6547809839248657,
+      "logits/rejected": -1.4826760292053223,
+      "logps/chosen": -434.3949279785156,
+      "logps/rejected": -483.05657958984375,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.691422462463379,
+      "rewards/margins": 0.7445409297943115,
+      "rewards/rejected": -2.4359633922576904,
+      "step": 640
+    },
+    {
+      "epoch": 0.17011253598534415,
+      "grad_norm": 16.53241539001465,
+      "learning_rate": 4.925960218232073e-06,
+      "logits/chosen": -1.586260199546814,
+      "logits/rejected": -1.4878358840942383,
+      "logps/chosen": -451.49871826171875,
+      "logps/rejected": -535.3369750976562,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9628738164901733,
+      "rewards/margins": 0.9344732165336609,
+      "rewards/rejected": -2.8973469734191895,
+      "step": 650
+    },
+    {
+      "epoch": 0.17272965192358022,
+      "grad_norm": 12.95315170288086,
+      "learning_rate": 4.920340499234116e-06,
+      "logits/chosen": -1.5459370613098145,
+      "logits/rejected": -1.3826103210449219,
+      "logps/chosen": -457.12640380859375,
+      "logps/rejected": -501.2940368652344,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.9127668142318726,
+      "rewards/margins": 0.7934447526931763,
+      "rewards/rejected": -2.706211566925049,
+      "step": 660
+    },
+    {
+      "epoch": 0.17534676786181627,
+      "grad_norm": 13.084680557250977,
+      "learning_rate": 4.914518682500995e-06,
+      "logits/chosen": -1.6595103740692139,
+      "logits/rejected": -1.5149381160736084,
+      "logps/chosen": -505.25360107421875,
+      "logps/rejected": -557.6201782226562,
+      "loss": 0.5516,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.2182066440582275,
+      "rewards/margins": 0.930493950843811,
+      "rewards/rejected": -3.1487009525299072,
+      "step": 670
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 9.387535095214844,
+      "learning_rate": 4.9084952541527315e-06,
+      "logits/chosen": -1.5597246885299683,
+      "logits/rejected": -1.4202911853790283,
+      "logps/chosen": -468.02777099609375,
+      "logps/rejected": -508.26483154296875,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.9475713968276978,
+      "rewards/margins": 0.8734270334243774,
+      "rewards/rejected": -2.820998430252075,
+      "step": 680
+    },
+    {
+      "epoch": 0.1805809997382884,
+      "grad_norm": 10.536702156066895,
+      "learning_rate": 4.902270717143858e-06,
+      "logits/chosen": -1.6048400402069092,
+      "logits/rejected": -1.5331923961639404,
+      "logps/chosen": -395.47686767578125,
+      "logps/rejected": -517.7385864257812,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.5932337045669556,
+      "rewards/margins": 1.1062676906585693,
+      "rewards/rejected": -2.6995015144348145,
+      "step": 690
+    },
+    {
+      "epoch": 0.18319811567652447,
+      "grad_norm": 6.061295509338379,
+      "learning_rate": 4.895845591221427e-06,
+      "logits/chosen": -1.5165598392486572,
+      "logits/rejected": -1.4684141874313354,
+      "logps/chosen": -450.1396484375,
+      "logps/rejected": -525.9497680664062,
+      "loss": 0.5394,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9692981243133545,
+      "rewards/margins": 0.802649199962616,
+      "rewards/rejected": -2.7719473838806152,
+      "step": 700
+    },
+    {
+      "epoch": 0.18319811567652447,
+      "eval_logits/chosen": -1.271895408630371,
+      "eval_logits/rejected": -1.1628035306930542,
+      "eval_logps/chosen": -491.6012268066406,
+      "eval_logps/rejected": -551.1991577148438,
+      "eval_loss": 0.5473812222480774,
+      "eval_rewards/accuracies": 0.7039999961853027,
+      "eval_rewards/chosen": -2.2699923515319824,
+      "eval_rewards/margins": 0.7962960004806519,
+      "eval_rewards/rejected": -3.066288471221924,
+      "eval_runtime": 1597.7171,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 700
+    },
+    {
+      "epoch": 0.18581523161476055,
+      "grad_norm": 13.924752235412598,
+      "learning_rate": 4.8892204128816e-06,
+      "logits/chosen": -1.3581098318099976,
+      "logits/rejected": -1.2599608898162842,
+      "logps/chosen": -515.8197631835938,
+      "logps/rejected": -587.2666625976562,
+      "loss": 0.5115,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.4833767414093018,
+      "rewards/margins": 0.8504128456115723,
+      "rewards/rejected": -3.333789348602295,
+      "step": 710
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 9.558119773864746,
+      "learning_rate": 4.882395735324864e-06,
+      "logits/chosen": -1.140836477279663,
+      "logits/rejected": -0.9971574544906616,
+      "logps/chosen": -565.4805908203125,
+      "logps/rejected": -648.9151611328125,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.9943554401397705,
+      "rewards/margins": 1.006219744682312,
+      "rewards/rejected": -4.000575065612793,
+      "step": 720
+    },
+    {
+      "epoch": 0.19104946349123267,
+      "grad_norm": 16.967636108398438,
+      "learning_rate": 4.87537212840983e-06,
+      "logits/chosen": -1.0284086465835571,
+      "logits/rejected": -0.9112738370895386,
+      "logps/chosen": -624.4201049804688,
+      "logps/rejected": -673.4655151367188,
+      "loss": 0.6183,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.582063674926758,
+      "rewards/margins": 0.7967410087585449,
+      "rewards/rejected": -4.378804683685303,
+      "step": 730
+    },
+    {
+      "epoch": 0.19366657942946872,
+      "grad_norm": 17.04477310180664,
+      "learning_rate": 4.8681501786056545e-06,
+      "logits/chosen": -1.1650705337524414,
+      "logits/rejected": -1.0339478254318237,
+      "logps/chosen": -438.68585205078125,
+      "logps/rejected": -490.553955078125,
+      "loss": 0.497,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.1054441928863525,
+      "rewards/margins": 0.8412445187568665,
+      "rewards/rejected": -2.946688652038574,
+      "step": 740
+    },
+    {
+      "epoch": 0.1962836953677048,
+      "grad_norm": 20.108728408813477,
+      "learning_rate": 4.860730488943068e-06,
+      "logits/chosen": -1.2052314281463623,
+      "logits/rejected": -1.157947301864624,
+      "logps/chosen": -439.97589111328125,
+      "logps/rejected": -518.9736328125,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.0552916526794434,
+      "rewards/margins": 0.809428870677948,
+      "rewards/rejected": -2.864720106124878,
+      "step": 750
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 6.034134387969971,
+      "learning_rate": 4.853113678964022e-06,
+      "logits/chosen": -1.3641645908355713,
+      "logits/rejected": -1.3035409450531006,
+      "logps/chosen": -421.71343994140625,
+      "logps/rejected": -494.55810546875,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4936286211013794,
+      "rewards/margins": 0.8079965710639954,
+      "rewards/rejected": -2.3016250133514404,
+      "step": 760
+    },
+    {
+      "epoch": 0.20151792724417691,
+      "grad_norm": 5.774670124053955,
+      "learning_rate": 4.845300384669958e-06,
+      "logits/chosen": -1.4722392559051514,
+      "logits/rejected": -1.3768599033355713,
+      "logps/chosen": -374.771484375,
+      "logps/rejected": -418.16656494140625,
+      "loss": 0.5331,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2165186405181885,
+      "rewards/margins": 0.6629087328910828,
+      "rewards/rejected": -1.8794273138046265,
+      "step": 770
+    },
+    {
+      "epoch": 0.204135043182413,
+      "grad_norm": 8.666418075561523,
+      "learning_rate": 4.837291258468701e-06,
+      "logits/chosen": -1.63046395778656,
+      "logits/rejected": -1.5205990076065063,
+      "logps/chosen": -421.2227478027344,
+      "logps/rejected": -473.2460021972656,
+      "loss": 0.5528,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3603990077972412,
+      "rewards/margins": 0.7691534161567688,
+      "rewards/rejected": -2.1295523643493652,
+      "step": 780
+    },
+    {
+      "epoch": 0.20675215912064904,
+      "grad_norm": 15.76352310180664,
+      "learning_rate": 4.829086969119984e-06,
+      "logits/chosen": -1.4890462160110474,
+      "logits/rejected": -1.4975069761276245,
+      "logps/chosen": -429.3702087402344,
+      "logps/rejected": -498.40496826171875,
+      "loss": 0.5898,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7876300811767578,
+      "rewards/margins": 0.6727088093757629,
+      "rewards/rejected": -2.460339069366455,
+      "step": 790
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 14.872965812683105,
+      "learning_rate": 4.820688201679605e-06,
+      "logits/chosen": -1.6982934474945068,
+      "logits/rejected": -1.4959887266159058,
+      "logps/chosen": -433.38470458984375,
+      "logps/rejected": -451.4923400878906,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.708482027053833,
+      "rewards/margins": 0.8051029443740845,
+      "rewards/rejected": -2.513584852218628,
+      "step": 800
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "eval_logits/chosen": -1.599565863609314,
+      "eval_logits/rejected": -1.510375738143921,
+      "eval_logps/chosen": -420.76544189453125,
+      "eval_logps/rejected": -474.2269287109375,
+      "eval_loss": 0.5322815179824829,
+      "eval_rewards/accuracies": 0.7225000262260437,
+      "eval_rewards/chosen": -1.5616350173950195,
+      "eval_rewards/margins": 0.7349306344985962,
+      "eval_rewards/rejected": -2.2965660095214844,
+      "eval_runtime": 1595.9174,
+      "eval_samples_per_second": 1.253,
+      "eval_steps_per_second": 0.157,
+      "step": 800
+    },
+    {
+      "epoch": 0.21198639099712116,
+      "grad_norm": 6.158263683319092,
+      "learning_rate": 4.8120956574422315e-06,
+      "logits/chosen": -1.8132537603378296,
+      "logits/rejected": -1.8140255212783813,
+      "logps/chosen": -400.74359130859375,
+      "logps/rejected": -452.96392822265625,
+      "loss": 0.58,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2691619396209717,
+      "rewards/margins": 0.5926799178123474,
+      "rewards/rejected": -1.8618419170379639,
+      "step": 810
+    },
+    {
+      "epoch": 0.21460350693535724,
+      "grad_norm": 14.991472244262695,
+      "learning_rate": 4.803310053882831e-06,
+      "logits/chosen": -1.883763313293457,
+      "logits/rejected": -1.9024746417999268,
+      "logps/chosen": -327.5868225097656,
+      "logps/rejected": -404.1509704589844,
+      "loss": 0.5258,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9834963083267212,
+      "rewards/margins": 0.6419707536697388,
+      "rewards/rejected": -1.6254669427871704,
+      "step": 820
+    },
+    {
+      "epoch": 0.2172206228735933,
+      "grad_norm": 10.492687225341797,
+      "learning_rate": 4.794332124596775e-06,
+      "logits/chosen": -1.8571357727050781,
+      "logits/rejected": -1.813534140586853,
+      "logps/chosen": -396.38372802734375,
+      "logps/rejected": -451.81634521484375,
+      "loss": 0.5868,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2799537181854248,
+      "rewards/margins": 0.5913030505180359,
+      "rewards/rejected": -1.8712568283081055,
+      "step": 830
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 10.114534378051758,
+      "learning_rate": 4.785162619238575e-06,
+      "logits/chosen": -1.8008487224578857,
+      "logits/rejected": -1.698720932006836,
+      "logps/chosen": -378.2573547363281,
+      "logps/rejected": -424.20782470703125,
+      "loss": 0.5402,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.244909405708313,
+      "rewards/margins": 0.7323002815246582,
+      "rewards/rejected": -1.9772096872329712,
+      "step": 840
+    },
+    {
+      "epoch": 0.22245485475006543,
+      "grad_norm": 7.237858295440674,
+      "learning_rate": 4.775802303459288e-06,
+      "logits/chosen": -1.6909987926483154,
+      "logits/rejected": -1.64451003074646,
+      "logps/chosen": -384.024169921875,
+      "logps/rejected": -455.19024658203125,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3631868362426758,
+      "rewards/margins": 0.7588311433792114,
+      "rewards/rejected": -2.1220178604125977,
+      "step": 850
+    },
+    {
+      "epoch": 0.22507197068830148,
+      "grad_norm": 20.08100700378418,
+      "learning_rate": 4.766251958842589e-06,
+      "logits/chosen": -1.5939120054244995,
+      "logits/rejected": -1.5511482954025269,
+      "logps/chosen": -447.6766662597656,
+      "logps/rejected": -501.3028259277344,
+      "loss": 0.569,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7521028518676758,
+      "rewards/margins": 0.6410677433013916,
+      "rewards/rejected": -2.3931705951690674,
+      "step": 860
+    },
+    {
+      "epoch": 0.22768908662653756,
+      "grad_norm": 8.703922271728516,
+      "learning_rate": 4.7565123828395066e-06,
+      "logits/chosen": -1.5731687545776367,
+      "logits/rejected": -1.5117802619934082,
+      "logps/chosen": -424.31463623046875,
+      "logps/rejected": -501.74725341796875,
+      "loss": 0.5338,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6849206686019897,
+      "rewards/margins": 0.7798541784286499,
+      "rewards/rejected": -2.4647748470306396,
+      "step": 870
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 7.03041410446167,
+      "learning_rate": 4.746584388701831e-06,
+      "logits/chosen": -1.6322410106658936,
+      "logits/rejected": -1.613351583480835,
+      "logps/chosen": -416.93463134765625,
+      "logps/rejected": -476.6209411621094,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.5381996631622314,
+      "rewards/margins": 0.7384254932403564,
+      "rewards/rejected": -2.276625156402588,
+      "step": 880
+    },
+    {
+      "epoch": 0.23292331850300968,
+      "grad_norm": 7.439998149871826,
+      "learning_rate": 4.736468805414218e-06,
+      "logits/chosen": -1.6738321781158447,
+      "logits/rejected": -1.6679372787475586,
+      "logps/chosen": -351.4508056640625,
+      "logps/rejected": -435.82196044921875,
+      "loss": 0.546,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0406602621078491,
+      "rewards/margins": 0.7058707475662231,
+      "rewards/rejected": -1.7465311288833618,
+      "step": 890
+    },
+    {
+      "epoch": 0.23554043444124576,
+      "grad_norm": 14.334321975708008,
+      "learning_rate": 4.7261664776249595e-06,
+      "logits/chosen": -1.5745285749435425,
+      "logits/rejected": -1.511311650276184,
+      "logps/chosen": -331.2271423339844,
+      "logps/rejected": -421.39752197265625,
+      "loss": 0.4763,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.024084210395813,
+      "rewards/margins": 0.9915010333061218,
+      "rewards/rejected": -2.015585422515869,
+      "step": 900
+    },
+    {
+      "epoch": 0.23554043444124576,
+      "eval_logits/chosen": -1.4989054203033447,
+      "eval_logits/rejected": -1.415571928024292,
+      "eval_logps/chosen": -425.90301513671875,
+      "eval_logps/rejected": -485.7890319824219,
+      "eval_loss": 0.5385720133781433,
+      "eval_rewards/accuracies": 0.7160000205039978,
+      "eval_rewards/chosen": -1.6130101680755615,
+      "eval_rewards/margins": 0.7991763353347778,
+      "eval_rewards/rejected": -2.412186622619629,
+      "eval_runtime": 1597.0973,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 900
+    },
+    {
+      "epoch": 0.2381575503794818,
+      "grad_norm": 12.946525573730469,
+      "learning_rate": 4.715678265575463e-06,
+      "logits/chosen": -1.5601496696472168,
+      "logits/rejected": -1.4170053005218506,
+      "logps/chosen": -460.2450256347656,
+      "logps/rejected": -471.73272705078125,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.759927749633789,
+      "rewards/margins": 0.6714185476303101,
+      "rewards/rejected": -2.4313464164733887,
+      "step": 910
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 7.0246663093566895,
+      "learning_rate": 4.705005045028415e-06,
+      "logits/chosen": -1.5280827283859253,
+      "logits/rejected": -1.4348738193511963,
+      "logps/chosen": -420.05682373046875,
+      "logps/rejected": -474.1004333496094,
+      "loss": 0.5575,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5702658891677856,
+      "rewards/margins": 0.7144767642021179,
+      "rewards/rejected": -2.284742832183838,
+      "step": 920
+    },
+    {
+      "epoch": 0.24339178225595393,
+      "grad_norm": 10.933419227600098,
+      "learning_rate": 4.694147707194659e-06,
+      "logits/chosen": -1.7351709604263306,
+      "logits/rejected": -1.6752073764801025,
+      "logps/chosen": -409.02301025390625,
+      "logps/rejected": -459.7960510253906,
+      "loss": 0.5172,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3725465536117554,
+      "rewards/margins": 0.7059783935546875,
+      "rewards/rejected": -2.0785250663757324,
+      "step": 930
+    },
+    {
+      "epoch": 0.24600889819419,
+      "grad_norm": 9.32016372680664,
+      "learning_rate": 4.683107158658782e-06,
+      "logits/chosen": -1.6696386337280273,
+      "logits/rejected": -1.5904427766799927,
+      "logps/chosen": -443.6543884277344,
+      "logps/rejected": -504.19293212890625,
+      "loss": 0.4894,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5346601009368896,
+      "rewards/margins": 0.8928316235542297,
+      "rewards/rejected": -2.4274916648864746,
+      "step": 940
+    },
+    {
+      "epoch": 0.24862601413242608,
+      "grad_norm": 15.52425765991211,
+      "learning_rate": 4.671884321303407e-06,
+      "logits/chosen": -1.7015752792358398,
+      "logits/rejected": -1.627673864364624,
+      "logps/chosen": -428.1681213378906,
+      "logps/rejected": -498.12530517578125,
+      "loss": 0.5229,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8192806243896484,
+      "rewards/margins": 0.843016505241394,
+      "rewards/rejected": -2.662297248840332,
+      "step": 950
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 9.358636856079102,
+      "learning_rate": 4.660480132232224e-06,
+      "logits/chosen": -1.7376630306243896,
+      "logits/rejected": -1.6711089611053467,
+      "logps/chosen": -425.44720458984375,
+      "logps/rejected": -465.4773864746094,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5372945070266724,
+      "rewards/margins": 0.6410431861877441,
+      "rewards/rejected": -2.178337812423706,
+      "step": 960
+    },
+    {
+      "epoch": 0.25386024600889817,
+      "grad_norm": 42.93048095703125,
+      "learning_rate": 4.6488955436917414e-06,
+      "logits/chosen": -1.6823867559432983,
+      "logits/rejected": -1.5265601873397827,
+      "logps/chosen": -445.4508361816406,
+      "logps/rejected": -489.4844665527344,
+      "loss": 0.53,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.655534029006958,
+      "rewards/margins": 0.9063774347305298,
+      "rewards/rejected": -2.5619113445281982,
+      "step": 970
+    },
+    {
+      "epoch": 0.2564773619471343,
+      "grad_norm": 10.477679252624512,
+      "learning_rate": 4.6371315229917644e-06,
+      "logits/chosen": -1.5334604978561401,
+      "logits/rejected": -1.4493004083633423,
+      "logps/chosen": -509.8636779785156,
+      "logps/rejected": -587.6615600585938,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.265300750732422,
+      "rewards/margins": 0.9821793437004089,
+      "rewards/rejected": -3.2474799156188965,
+      "step": 980
+    },
+    {
+      "epoch": 0.2590944778853703,
+      "grad_norm": 12.81888198852539,
+      "learning_rate": 4.625189052424638e-06,
+      "logits/chosen": -1.4550929069519043,
+      "logits/rejected": -1.353437066078186,
+      "logps/chosen": -502.6724548339844,
+      "logps/rejected": -594.8109130859375,
+      "loss": 0.4652,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.6365292072296143,
+      "rewards/margins": 1.1491750478744507,
+      "rewards/rejected": -3.7857041358947754,
+      "step": 990
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 12.723479270935059,
+      "learning_rate": 4.613069129183218e-06,
+      "logits/chosen": -1.5442698001861572,
+      "logits/rejected": -1.4192157983779907,
+      "logps/chosen": -524.4293212890625,
+      "logps/rejected": -580.075439453125,
+      "loss": 0.5266,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.2106876373291016,
+      "rewards/margins": 0.9188516736030579,
+      "rewards/rejected": -3.1295390129089355,
+      "step": 1000
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "eval_logits/chosen": -1.3050363063812256,
+      "eval_logits/rejected": -1.2043476104736328,
+      "eval_logps/chosen": -482.4830627441406,
+      "eval_logps/rejected": -550.0311279296875,
+      "eval_loss": 0.523389995098114,
+      "eval_rewards/accuracies": 0.7279999852180481,
+      "eval_rewards/chosen": -2.1788110733032227,
+      "eval_rewards/margins": 0.8757960796356201,
+      "eval_rewards/rejected": -3.0546071529388428,
+      "eval_runtime": 1595.4607,
+      "eval_samples_per_second": 1.254,
+      "eval_steps_per_second": 0.157,
+      "step": 1000
+    },
+    {
+      "epoch": 0.2643287097618425,
+      "grad_norm": 10.429312705993652,
+      "learning_rate": 4.600772765277607e-06,
+      "logits/chosen": -1.3304941654205322,
+      "logits/rejected": -1.263293743133545,
+      "logps/chosen": -446.70635986328125,
+      "logps/rejected": -535.6898193359375,
+      "loss": 0.4782,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.131920099258423,
+      "rewards/margins": 0.9416648745536804,
+      "rewards/rejected": -3.073584794998169,
+      "step": 1010
+    },
+    {
+      "epoch": 0.2669458257000785,
+      "grad_norm": 16.054344177246094,
+      "learning_rate": 4.588300987450652e-06,
+      "logits/chosen": -1.4466055631637573,
+      "logits/rejected": -1.3539087772369385,
+      "logps/chosen": -452.9022521972656,
+      "logps/rejected": -500.742919921875,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9927232265472412,
+      "rewards/margins": 0.8132057189941406,
+      "rewards/rejected": -2.8059287071228027,
+      "step": 1020
+    },
+    {
+      "epoch": 0.26956294163831457,
+      "grad_norm": 7.860910415649414,
+      "learning_rate": 4.5756548370922136e-06,
+      "logits/chosen": -1.514672875404358,
+      "logits/rejected": -1.447291374206543,
+      "logps/chosen": -394.26031494140625,
+      "logps/rejected": -461.3570251464844,
+      "loss": 0.5174,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.536415934562683,
+      "rewards/margins": 0.7608687281608582,
+      "rewards/rejected": -2.2972846031188965,
+      "step": 1030
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 13.006108283996582,
+      "learning_rate": 4.562835370152206e-06,
+      "logits/chosen": -1.559757113456726,
+      "logits/rejected": -1.4329888820648193,
+      "logps/chosen": -482.7850646972656,
+      "logps/rejected": -568.3600463867188,
+      "loss": 0.4777,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8161367177963257,
+      "rewards/margins": 1.1671006679534912,
+      "rewards/rejected": -2.9832375049591064,
+      "step": 1040
+    },
+    {
+      "epoch": 0.2747971735147867,
+      "grad_norm": 10.500707626342773,
+      "learning_rate": 4.54984365705243e-06,
+      "logits/chosen": -1.5195045471191406,
+      "logits/rejected": -1.4307035207748413,
+      "logps/chosen": -455.4422912597656,
+      "logps/rejected": -578.11181640625,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8703197240829468,
+      "rewards/margins": 1.292755126953125,
+      "rewards/rejected": -3.1630749702453613,
+      "step": 1050
+    },
+    {
+      "epoch": 0.27741428945302277,
+      "grad_norm": 8.591322898864746,
+      "learning_rate": 4.536680782597191e-06,
+      "logits/chosen": -1.5150272846221924,
+      "logits/rejected": -1.4561691284179688,
+      "logps/chosen": -402.49053955078125,
+      "logps/rejected": -479.2098083496094,
+      "loss": 0.5844,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6649020910263062,
+      "rewards/margins": 0.8239375352859497,
+      "rewards/rejected": -2.488839626312256,
+      "step": 1060
+    },
+    {
+      "epoch": 0.2800314053912588,
+      "grad_norm": 12.471731185913086,
+      "learning_rate": 4.523347845882718e-06,
+      "logits/chosen": -1.598962664604187,
+      "logits/rejected": -1.4552241563796997,
+      "logps/chosen": -423.9444274902344,
+      "logps/rejected": -476.2308654785156,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.3588608503341675,
+      "rewards/margins": 1.0158392190933228,
+      "rewards/rejected": -2.3747003078460693,
+      "step": 1070
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 8.733776092529297,
+      "learning_rate": 4.50984596020539e-06,
+      "logits/chosen": -1.2865254878997803,
+      "logits/rejected": -1.242490530014038,
+      "logps/chosen": -467.01776123046875,
+      "logps/rejected": -511.63507080078125,
+      "loss": 0.5623,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8692843914031982,
+      "rewards/margins": 0.7691918611526489,
+      "rewards/rejected": -2.6384763717651367,
+      "step": 1080
+    },
+    {
+      "epoch": 0.28526563726773096,
+      "grad_norm": 9.051119804382324,
+      "learning_rate": 4.4961762529687745e-06,
+      "logits/chosen": -1.2762900590896606,
+      "logits/rejected": -1.1670513153076172,
+      "logps/chosen": -481.23406982421875,
+      "logps/rejected": -555.0721435546875,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.2087130546569824,
+      "rewards/margins": 0.9038770794868469,
+      "rewards/rejected": -3.1125903129577637,
+      "step": 1090
+    },
+    {
+      "epoch": 0.287882753205967,
+      "grad_norm": 7.552842617034912,
+      "learning_rate": 4.482339865589492e-06,
+      "logits/chosen": -1.240541934967041,
+      "logits/rejected": -1.0851424932479858,
+      "logps/chosen": -478.9158630371094,
+      "logps/rejected": -492.19097900390625,
+      "loss": 0.59,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.127556562423706,
+      "rewards/margins": 0.5717890858650208,
+      "rewards/rejected": -2.699345827102661,
+      "step": 1100
+    },
+    {
+      "epoch": 0.287882753205967,
+      "eval_logits/chosen": -1.1099953651428223,
+      "eval_logits/rejected": -0.9899115562438965,
+      "eval_logps/chosen": -433.97100830078125,
+      "eval_logps/rejected": -478.8385314941406,
+      "eval_loss": 0.5277644991874695,
+      "eval_rewards/accuracies": 0.7300000190734863,
+      "eval_rewards/chosen": -1.693690299987793,
+      "eval_rewards/margins": 0.6489914059638977,
+      "eval_rewards/rejected": -2.342681646347046,
+      "eval_runtime": 1596.8217,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 1100
+    },
+    {
+      "epoch": 0.2904998691442031,
+      "grad_norm": 5.962321758270264,
+      "learning_rate": 4.468337953401909e-06,
+      "logits/chosen": -1.3310126066207886,
+      "logits/rejected": -1.2818472385406494,
+      "logps/chosen": -426.1422424316406,
+      "logps/rejected": -471.98175048828125,
+      "loss": 0.572,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5579837560653687,
+      "rewards/margins": 0.5076408982276917,
+      "rewards/rejected": -2.065624713897705,
+      "step": 1110
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 5.4544477462768555,
+      "learning_rate": 4.45417168556166e-06,
+      "logits/chosen": -1.3914196491241455,
+      "logits/rejected": -1.305474877357483,
+      "logps/chosen": -344.81463623046875,
+      "logps/rejected": -411.66412353515625,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0368053913116455,
+      "rewards/margins": 0.6763306856155396,
+      "rewards/rejected": -1.7131359577178955,
+      "step": 1120
+    },
+    {
+      "epoch": 0.2957341010206752,
+      "grad_norm": 12.404394149780273,
+      "learning_rate": 4.439842244948036e-06,
+      "logits/chosen": -1.3826755285263062,
+      "logits/rejected": -1.251558780670166,
+      "logps/chosen": -408.0195007324219,
+      "logps/rejected": -476.11114501953125,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4701926708221436,
+      "rewards/margins": 0.7369771003723145,
+      "rewards/rejected": -2.207169771194458,
+      "step": 1130
+    },
+    {
+      "epoch": 0.29835121695891126,
+      "grad_norm": 28.483901977539062,
+      "learning_rate": 4.425350828065204e-06,
+      "logits/chosen": -1.342377781867981,
+      "logits/rejected": -1.165569543838501,
+      "logps/chosen": -453.5723571777344,
+      "logps/rejected": -509.80731201171875,
+      "loss": 0.4798,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.6950212717056274,
+      "rewards/margins": 1.041156530380249,
+      "rewards/rejected": -2.736177921295166,
+      "step": 1140
+    },
+    {
+      "epoch": 0.30096833289714736,
+      "grad_norm": 9.420351028442383,
+      "learning_rate": 4.410698644942303e-06,
+      "logits/chosen": -1.3900500535964966,
+      "logits/rejected": -1.2638782262802124,
+      "logps/chosen": -445.68646240234375,
+      "logps/rejected": -537.0733642578125,
+      "loss": 0.4689,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.7342853546142578,
+      "rewards/margins": 1.1538056135177612,
+      "rewards/rejected": -2.8880913257598877,
+      "step": 1150
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 11.15323543548584,
+      "learning_rate": 4.395886919032406e-06,
+      "logits/chosen": -1.3719291687011719,
+      "logits/rejected": -1.2517584562301636,
+      "logps/chosen": -444.84375,
+      "logps/rejected": -515.7125244140625,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8111295700073242,
+      "rewards/margins": 0.9737447500228882,
+      "rewards/rejected": -2.784874439239502,
+      "step": 1160
+    },
+    {
+      "epoch": 0.30620256477361946,
+      "grad_norm": 11.627602577209473,
+      "learning_rate": 4.380916887110366e-06,
+      "logits/chosen": -1.5677311420440674,
+      "logits/rejected": -1.4189679622650146,
+      "logps/chosen": -418.11309814453125,
+      "logps/rejected": -474.2850646972656,
+      "loss": 0.5335,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5903016328811646,
+      "rewards/margins": 0.9656845331192017,
+      "rewards/rejected": -2.555985927581787,
+      "step": 1170
+    },
+    {
+      "epoch": 0.30881968071185556,
+      "grad_norm": 10.718832969665527,
+      "learning_rate": 4.365789799169539e-06,
+      "logits/chosen": -1.2974934577941895,
+      "logits/rejected": -1.3446776866912842,
+      "logps/chosen": -414.12152099609375,
+      "logps/rejected": -495.92340087890625,
+      "loss": 0.5394,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6758350133895874,
+      "rewards/margins": 0.7650316953659058,
+      "rewards/rejected": -2.440866708755493,
+      "step": 1180
+    },
+    {
+      "epoch": 0.3114367966500916,
+      "grad_norm": 8.005572319030762,
+      "learning_rate": 4.350506918317416e-06,
+      "logits/chosen": -1.4092941284179688,
+      "logits/rejected": -1.264432668685913,
+      "logps/chosen": -408.0471496582031,
+      "logps/rejected": -483.52960205078125,
+      "loss": 0.5148,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.605602502822876,
+      "rewards/margins": 0.8261173963546753,
+      "rewards/rejected": -2.431720018386841,
+      "step": 1190
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 7.756717681884766,
+      "learning_rate": 4.335069520670149e-06,
+      "logits/chosen": -1.2672417163848877,
+      "logits/rejected": -1.1932761669158936,
+      "logps/chosen": -391.0654602050781,
+      "logps/rejected": -470.7328186035156,
+      "loss": 0.5724,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6141713857650757,
+      "rewards/margins": 0.7146965265274048,
+      "rewards/rejected": -2.3288679122924805,
+      "step": 1200
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "eval_logits/chosen": -1.2473385334014893,
+      "eval_logits/rejected": -1.1348552703857422,
+      "eval_logps/chosen": -420.0863342285156,
+      "eval_logps/rejected": -485.28948974609375,
+      "eval_loss": 0.5071337819099426,
+      "eval_rewards/accuracies": 0.7379999756813049,
+      "eval_rewards/chosen": -1.5548440217971802,
+      "eval_rewards/margins": 0.8523474335670471,
+      "eval_rewards/rejected": -2.407191514968872,
+      "eval_runtime": 1597.3419,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 1200
+    },
+    {
+      "epoch": 0.3166710285265637,
+      "grad_norm": 5.270259857177734,
+      "learning_rate": 4.319478895246e-06,
+      "logits/chosen": -1.3534616231918335,
+      "logits/rejected": -1.1892000436782837,
+      "logps/chosen": -396.1217346191406,
+      "logps/rejected": -454.44378662109375,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.490412950515747,
+      "rewards/margins": 0.8302758932113647,
+      "rewards/rejected": -2.3206887245178223,
+      "step": 1210
+    },
+    {
+      "epoch": 0.3192881444647998,
+      "grad_norm": 12.79388427734375,
+      "learning_rate": 4.303736343857704e-06,
+      "logits/chosen": -1.3265999555587769,
+      "logits/rejected": -1.2219207286834717,
+      "logps/chosen": -440.6163024902344,
+      "logps/rejected": -568.152099609375,
+      "loss": 0.4921,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.835211992263794,
+      "rewards/margins": 1.1568537950515747,
+      "rewards/rejected": -2.992065906524658,
+      "step": 1220
+    },
+    {
+      "epoch": 0.32190526040303585,
+      "grad_norm": 13.922798156738281,
+      "learning_rate": 4.287843181003772e-06,
+      "logits/chosen": -1.3599532842636108,
+      "logits/rejected": -1.237168550491333,
+      "logps/chosen": -486.5572204589844,
+      "logps/rejected": -522.0225219726562,
+      "loss": 0.5566,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9279435873031616,
+      "rewards/margins": 0.8177730441093445,
+      "rewards/rejected": -2.7457165718078613,
+      "step": 1230
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 7.278261184692383,
+      "learning_rate": 4.27180073375873e-06,
+      "logits/chosen": -1.3576252460479736,
+      "logits/rejected": -1.2608816623687744,
+      "logps/chosen": -445.9983825683594,
+      "logps/rejected": -495.5048828125,
+      "loss": 0.4942,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.587226152420044,
+      "rewards/margins": 0.910873293876648,
+      "rewards/rejected": -2.4980995655059814,
+      "step": 1240
+    },
+    {
+      "epoch": 0.327139492279508,
+      "grad_norm": 8.15560531616211,
+      "learning_rate": 4.255610341662304e-06,
+      "logits/chosen": -1.3735462427139282,
+      "logits/rejected": -1.210055947303772,
+      "logps/chosen": -444.36285400390625,
+      "logps/rejected": -524.8583374023438,
+      "loss": 0.5542,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8914082050323486,
+      "rewards/margins": 1.0044190883636475,
+      "rewards/rejected": -2.895827293395996,
+      "step": 1250
+    },
+    {
+      "epoch": 0.32975660821774405,
+      "grad_norm": 10.175187110900879,
+      "learning_rate": 4.2392733566075764e-06,
+      "logits/chosen": -1.4199110269546509,
+      "logits/rejected": -1.317662000656128,
+      "logps/chosen": -492.12139892578125,
+      "logps/rejected": -561.2523803710938,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.361861228942871,
+      "rewards/margins": 0.8468947410583496,
+      "rewards/rejected": -3.2087559700012207,
+      "step": 1260
+    },
+    {
+      "epoch": 0.3323737241559801,
+      "grad_norm": 8.98975944519043,
+      "learning_rate": 4.2227911427280975e-06,
+      "logits/chosen": -1.5209752321243286,
+      "logits/rejected": -1.3899322748184204,
+      "logps/chosen": -398.7652282714844,
+      "logps/rejected": -442.14892578125,
+      "loss": 0.5594,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4862077236175537,
+      "rewards/margins": 0.761127769947052,
+      "rewards/rejected": -2.247335195541382,
+      "step": 1270
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 9.442636489868164,
+      "learning_rate": 4.206165076283983e-06,
+      "logits/chosen": -1.5363355875015259,
+      "logits/rejected": -1.428397536277771,
+      "logps/chosen": -410.4297790527344,
+      "logps/rejected": -474.6888122558594,
+      "loss": 0.5101,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6462541818618774,
+      "rewards/margins": 0.8527911305427551,
+      "rewards/rejected": -2.4990451335906982,
+      "step": 1280
+    },
+    {
+      "epoch": 0.33760795603245225,
+      "grad_norm": 9.517970085144043,
+      "learning_rate": 4.189396545546995e-06,
+      "logits/chosen": -1.4206923246383667,
+      "logits/rejected": -1.3242676258087158,
+      "logps/chosen": -455.42474365234375,
+      "logps/rejected": -532.8533325195312,
+      "loss": 0.5055,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.0629706382751465,
+      "rewards/margins": 0.9739512205123901,
+      "rewards/rejected": -3.036921977996826,
+      "step": 1290
+    },
+    {
+      "epoch": 0.3402250719706883,
+      "grad_norm": 11.92896842956543,
+      "learning_rate": 4.172486950684627e-06,
+      "logits/chosen": -1.458961844444275,
+      "logits/rejected": -1.4081146717071533,
+      "logps/chosen": -429.62548828125,
+      "logps/rejected": -503.53582763671875,
+      "loss": 0.5457,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7603120803833008,
+      "rewards/margins": 0.771033525466919,
+      "rewards/rejected": -2.5313456058502197,
+      "step": 1300
+    },
+    {
+      "epoch": 0.3402250719706883,
+      "eval_logits/chosen": -1.3403185606002808,
+      "eval_logits/rejected": -1.242436408996582,
+      "eval_logps/chosen": -440.03851318359375,
+      "eval_logps/rejected": -507.2138366699219,
+      "eval_loss": 0.5013459920883179,
+      "eval_rewards/accuracies": 0.7434999942779541,
+      "eval_rewards/chosen": -1.7543656826019287,
+      "eval_rewards/margins": 0.8720693588256836,
+      "eval_rewards/rejected": -2.6264350414276123,
+      "eval_runtime": 1597.2746,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 1300
+    },
+    {
+      "epoch": 0.34284218790892435,
+      "grad_norm": 7.771608352661133,
+      "learning_rate": 4.155437703643182e-06,
+      "logits/chosen": -1.5240622758865356,
+      "logits/rejected": -1.3838030099868774,
+      "logps/chosen": -408.34857177734375,
+      "logps/rejected": -462.7867736816406,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6171270608901978,
+      "rewards/margins": 0.8352136611938477,
+      "rewards/rejected": -2.452340602874756,
+      "step": 1310
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 13.282800674438477,
+      "learning_rate": 4.138250228029882e-06,
+      "logits/chosen": -1.4314453601837158,
+      "logits/rejected": -1.3674726486206055,
+      "logps/chosen": -449.40045166015625,
+      "logps/rejected": -546.0065307617188,
+      "loss": 0.5029,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.9253730773925781,
+      "rewards/margins": 0.9314772486686707,
+      "rewards/rejected": -2.8568501472473145,
+      "step": 1320
+    },
+    {
+      "epoch": 0.3480764197853965,
+      "grad_norm": 5.933595180511475,
+      "learning_rate": 4.120925958993994e-06,
+      "logits/chosen": -1.3017045259475708,
+      "logits/rejected": -1.251571536064148,
+      "logps/chosen": -431.25537109375,
+      "logps/rejected": -517.0537109375,
+      "loss": 0.557,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.9745893478393555,
+      "rewards/margins": 0.9014847874641418,
+      "rewards/rejected": -2.8760738372802734,
+      "step": 1330
+    },
+    {
+      "epoch": 0.35069353572363254,
+      "grad_norm": 11.526862144470215,
+      "learning_rate": 4.103466343106999e-06,
+      "logits/chosen": -1.3678683042526245,
+      "logits/rejected": -1.2736116647720337,
+      "logps/chosen": -471.77728271484375,
+      "logps/rejected": -537.7550048828125,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.0238394737243652,
+      "rewards/margins": 0.9198586344718933,
+      "rewards/rejected": -2.9436984062194824,
+      "step": 1340
+    },
+    {
+      "epoch": 0.35331065166186865,
+      "grad_norm": 10.23589038848877,
+      "learning_rate": 4.085872838241797e-06,
+      "logits/chosen": -1.3284608125686646,
+      "logits/rejected": -1.217164397239685,
+      "logps/chosen": -450.1647033691406,
+      "logps/rejected": -507.2977600097656,
+      "loss": 0.569,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8795936107635498,
+      "rewards/margins": 0.7713474631309509,
+      "rewards/rejected": -2.6509411334991455,
+      "step": 1350
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 9.470162391662598,
+      "learning_rate": 4.06814691345098e-06,
+      "logits/chosen": -1.387274980545044,
+      "logits/rejected": -1.2676749229431152,
+      "logps/chosen": -405.6870422363281,
+      "logps/rejected": -467.76708984375,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5023095607757568,
+      "rewards/margins": 0.8411477208137512,
+      "rewards/rejected": -2.3434574604034424,
+      "step": 1360
+    },
+    {
+      "epoch": 0.35854488353834074,
+      "grad_norm": 11.479103088378906,
+      "learning_rate": 4.050290048844171e-06,
+      "logits/chosen": -1.4606144428253174,
+      "logits/rejected": -1.400508165359497,
+      "logps/chosen": -422.5201721191406,
+      "logps/rejected": -496.3780212402344,
+      "loss": 0.5403,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5399492979049683,
+      "rewards/margins": 0.7900384068489075,
+      "rewards/rejected": -2.3299877643585205,
+      "step": 1370
+    },
+    {
+      "epoch": 0.3611619994765768,
+      "grad_norm": 9.197821617126465,
+      "learning_rate": 4.032303735464422e-06,
+      "logits/chosen": -1.6123807430267334,
+      "logits/rejected": -1.459542989730835,
+      "logps/chosen": -428.04095458984375,
+      "logps/rejected": -507.18243408203125,
+      "loss": 0.4608,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.5614588260650635,
+      "rewards/margins": 1.0217015743255615,
+      "rewards/rejected": -2.583160161972046,
+      "step": 1380
+    },
+    {
+      "epoch": 0.3637791154148129,
+      "grad_norm": 13.988523483276367,
+      "learning_rate": 4.014189475163727e-06,
+      "logits/chosen": -1.4627307653427124,
+      "logits/rejected": -1.3992432355880737,
+      "logps/chosen": -429.89642333984375,
+      "logps/rejected": -529.4677124023438,
+      "loss": 0.4813,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.753618836402893,
+      "rewards/margins": 1.0768356323242188,
+      "rewards/rejected": -2.8304543495178223,
+      "step": 1390
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 14.094236373901367,
+      "learning_rate": 3.995948780477605e-06,
+      "logits/chosen": -1.6445674896240234,
+      "logits/rejected": -1.5534647703170776,
+      "logps/chosen": -442.5335388183594,
+      "logps/rejected": -506.78253173828125,
+      "loss": 0.5423,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7385400533676147,
+      "rewards/margins": 0.8896828889846802,
+      "rewards/rejected": -2.628222942352295,
+      "step": 1400
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "eval_logits/chosen": -1.586852788925171,
+      "eval_logits/rejected": -1.5062702894210815,
+      "eval_logps/chosen": -428.40972900390625,
+      "eval_logps/rejected": -505.70770263671875,
+      "eval_loss": 0.5131703019142151,
+      "eval_rewards/accuracies": 0.7210000157356262,
+      "eval_rewards/chosen": -1.6380778551101685,
+      "eval_rewards/margins": 0.9732955098152161,
+      "eval_rewards/rejected": -2.6113734245300293,
+      "eval_runtime": 1597.5064,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 1400
+    },
+    {
+      "epoch": 0.369013347291285,
+      "grad_norm": 13.169591903686523,
+      "learning_rate": 3.977583174498816e-06,
+      "logits/chosen": -1.6033601760864258,
+      "logits/rejected": -1.529076099395752,
+      "logps/chosen": -446.4644470214844,
+      "logps/rejected": -562.0938720703125,
+      "loss": 0.4006,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8283302783966064,
+      "rewards/margins": 1.3763213157653809,
+      "rewards/rejected": -3.2046515941619873,
+      "step": 1410
+    },
+    {
+      "epoch": 0.3716304632295211,
+      "grad_norm": 12.735097885131836,
+      "learning_rate": 3.959094190750172e-06,
+      "logits/chosen": -1.6629726886749268,
+      "logits/rejected": -1.5727919340133667,
+      "logps/chosen": -460.08013916015625,
+      "logps/rejected": -542.1484375,
+      "loss": 0.5034,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7491388320922852,
+      "rewards/margins": 1.091073989868164,
+      "rewards/rejected": -2.8402130603790283,
+      "step": 1420
+    },
+    {
+      "epoch": 0.37424757916775714,
+      "grad_norm": 9.543456077575684,
+      "learning_rate": 3.9404833730564975e-06,
+      "logits/chosen": -1.5923559665679932,
+      "logits/rejected": -1.5246838331222534,
+      "logps/chosen": -397.0897216796875,
+      "logps/rejected": -477.6280212402344,
+      "loss": 0.5221,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4408881664276123,
+      "rewards/margins": 0.9207174181938171,
+      "rewards/rejected": -2.361605405807495,
+      "step": 1430
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 15.872808456420898,
+      "learning_rate": 3.921752275415712e-06,
+      "logits/chosen": -1.6502765417099,
+      "logits/rejected": -1.5927629470825195,
+      "logps/chosen": -403.17657470703125,
+      "logps/rejected": -513.6209106445312,
+      "loss": 0.4272,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.527881145477295,
+      "rewards/margins": 1.3093104362487793,
+      "rewards/rejected": -2.837191581726074,
+      "step": 1440
+    },
+    {
+      "epoch": 0.37948181104422923,
+      "grad_norm": 13.932242393493652,
+      "learning_rate": 3.902902461869079e-06,
+      "logits/chosen": -1.6036418676376343,
+      "logits/rejected": -1.5119550228118896,
+      "logps/chosen": -423.63275146484375,
+      "logps/rejected": -522.7612915039062,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8413972854614258,
+      "rewards/margins": 1.1561634540557861,
+      "rewards/rejected": -2.997560501098633,
+      "step": 1450
+    },
+    {
+      "epoch": 0.38209892698246534,
+      "grad_norm": 17.43979835510254,
+      "learning_rate": 3.883935506370605e-06,
+      "logits/chosen": -1.6155637502670288,
+      "logits/rejected": -1.546623706817627,
+      "logps/chosen": -433.8269958496094,
+      "logps/rejected": -493.6173400878906,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8069992065429688,
+      "rewards/margins": 0.8752773404121399,
+      "rewards/rejected": -2.682276487350464,
+      "step": 1460
+    },
+    {
+      "epoch": 0.3847160429207014,
+      "grad_norm": 6.709349155426025,
+      "learning_rate": 3.864852992655617e-06,
+      "logits/chosen": -1.6773513555526733,
+      "logits/rejected": -1.6156046390533447,
+      "logps/chosen": -408.98004150390625,
+      "logps/rejected": -496.5398864746094,
+      "loss": 0.4636,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.6042149066925049,
+      "rewards/margins": 0.9984095692634583,
+      "rewards/rejected": -2.6026244163513184,
+      "step": 1470
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 5.378371715545654,
+      "learning_rate": 3.845656514108516e-06,
+      "logits/chosen": -1.6842008829116821,
+      "logits/rejected": -1.5901457071304321,
+      "logps/chosen": -467.2499084472656,
+      "logps/rejected": -508.35491943359375,
+      "loss": 0.5207,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0820114612579346,
+      "rewards/margins": 0.9600709080696106,
+      "rewards/rejected": -3.0420823097229004,
+      "step": 1480
+    },
+    {
+      "epoch": 0.38995027479717354,
+      "grad_norm": 14.073667526245117,
+      "learning_rate": 3.826347673629738e-06,
+      "logits/chosen": -1.6913728713989258,
+      "logits/rejected": -1.5630801916122437,
+      "logps/chosen": -425.8887634277344,
+      "logps/rejected": -511.43194580078125,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7265421152114868,
+      "rewards/margins": 1.118648648262024,
+      "rewards/rejected": -2.84519100189209,
+      "step": 1490
+    },
+    {
+      "epoch": 0.3925673907354096,
+      "grad_norm": 15.713250160217285,
+      "learning_rate": 3.8069280835019062e-06,
+      "logits/chosen": -1.631744384765625,
+      "logits/rejected": -1.5060975551605225,
+      "logps/chosen": -430.0010681152344,
+      "logps/rejected": -537.3772583007812,
+      "loss": 0.4492,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.657666563987732,
+      "rewards/margins": 1.2653546333312988,
+      "rewards/rejected": -2.9230213165283203,
+      "step": 1500
+    },
+    {
+      "epoch": 0.3925673907354096,
+      "eval_logits/chosen": -1.5949609279632568,
+      "eval_logits/rejected": -1.4971818923950195,
+      "eval_logps/chosen": -423.41754150390625,
+      "eval_logps/rejected": -503.4827575683594,
+      "eval_loss": 0.5122300386428833,
+      "eval_rewards/accuracies": 0.7260000109672546,
+      "eval_rewards/chosen": -1.5881556272506714,
+      "eval_rewards/margins": 1.0009682178497314,
+      "eval_rewards/rejected": -2.5891237258911133,
+      "eval_runtime": 1597.6931,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 1500
+    },
+    {
+      "epoch": 0.39518450667364563,
+      "grad_norm": 14.919211387634277,
+      "learning_rate": 3.7873993652552077e-06,
+      "logits/chosen": -1.6476099491119385,
+      "logits/rejected": -1.5861533880233765,
+      "logps/chosen": -370.7307434082031,
+      "logps/rejected": -437.010986328125,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.370052695274353,
+      "rewards/margins": 0.7218869924545288,
+      "rewards/rejected": -2.091939926147461,
+      "step": 1510
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 6.893444061279297,
+      "learning_rate": 3.7677631495319953e-06,
+      "logits/chosen": -1.7386844158172607,
+      "logits/rejected": -1.6610181331634521,
+      "logps/chosen": -358.55792236328125,
+      "logps/rejected": -410.8779296875,
+      "loss": 0.5166,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9495410919189453,
+      "rewards/margins": 0.7035711407661438,
+      "rewards/rejected": -1.6531124114990234,
+      "step": 1520
+    },
+    {
+      "epoch": 0.4004187385501178,
+      "grad_norm": 10.669229507446289,
+      "learning_rate": 3.748021075950633e-06,
+      "logits/chosen": -1.6956592798233032,
+      "logits/rejected": -1.634338617324829,
+      "logps/chosen": -401.1066589355469,
+      "logps/rejected": -450.43701171875,
+      "loss": 0.5677,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.2650973796844482,
+      "rewards/margins": 0.5999857187271118,
+      "rewards/rejected": -1.8650833368301392,
+      "step": 1530
+    },
+    {
+      "epoch": 0.40303585448835383,
+      "grad_norm": 13.775323867797852,
+      "learning_rate": 3.7281747929685824e-06,
+      "logits/chosen": -1.4220095872879028,
+      "logits/rejected": -1.3044965267181396,
+      "logps/chosen": -428.1224060058594,
+      "logps/rejected": -506.9508361816406,
+      "loss": 0.5483,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.9053666591644287,
+      "rewards/margins": 0.9458476305007935,
+      "rewards/rejected": -2.8512144088745117,
+      "step": 1540
+    },
+    {
+      "epoch": 0.4056529704265899,
+      "grad_norm": 14.96325397491455,
+      "learning_rate": 3.7082259577447604e-06,
+      "logits/chosen": -1.4775934219360352,
+      "logits/rejected": -1.3828151226043701,
+      "logps/chosen": -484.23712158203125,
+      "logps/rejected": -562.1600341796875,
+      "loss": 0.4745,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.1311516761779785,
+      "rewards/margins": 0.9796358942985535,
+      "rewards/rejected": -3.1107876300811768,
+      "step": 1550
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 12.222273826599121,
+      "learning_rate": 3.6881762360011688e-06,
+      "logits/chosen": -1.5065914392471313,
+      "logits/rejected": -1.3439867496490479,
+      "logps/chosen": -507.56646728515625,
+      "logps/rejected": -579.8803100585938,
+      "loss": 0.5187,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.261667013168335,
+      "rewards/margins": 1.0928138494491577,
+      "rewards/rejected": -3.3544812202453613,
+      "step": 1560
+    },
+    {
+      "epoch": 0.410887202303062,
+      "grad_norm": 9.984075546264648,
+      "learning_rate": 3.668027301883802e-06,
+      "logits/chosen": -1.5400466918945312,
+      "logits/rejected": -1.42914617061615,
+      "logps/chosen": -428.8944396972656,
+      "logps/rejected": -516.1829833984375,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7761850357055664,
+      "rewards/margins": 1.021864891052246,
+      "rewards/rejected": -2.7980499267578125,
+      "step": 1570
+    },
+    {
+      "epoch": 0.4135043182412981,
+      "grad_norm": 8.096105575561523,
+      "learning_rate": 3.64778083782286e-06,
+      "logits/chosen": -1.4683252573013306,
+      "logits/rejected": -1.4568402767181396,
+      "logps/chosen": -420.28662109375,
+      "logps/rejected": -531.93310546875,
+      "loss": 0.5363,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.67800772190094,
+      "rewards/margins": 0.8264617919921875,
+      "rewards/rejected": -2.504469394683838,
+      "step": 1580
+    },
+    {
+      "epoch": 0.4161214341795342,
+      "grad_norm": 8.454867362976074,
+      "learning_rate": 3.627438534392268e-06,
+      "logits/chosen": -1.563232421875,
+      "logits/rejected": -1.5413362979888916,
+      "logps/chosen": -382.3138122558594,
+      "logps/rejected": -473.07427978515625,
+      "loss": 0.4969,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.4439641237258911,
+      "rewards/margins": 0.8402025103569031,
+      "rewards/rejected": -2.2841668128967285,
+      "step": 1590
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 8.18990421295166,
+      "learning_rate": 3.607002090168506e-06,
+      "logits/chosen": -1.374459981918335,
+      "logits/rejected": -1.3057047128677368,
+      "logps/chosen": -437.67437744140625,
+      "logps/rejected": -498.42022705078125,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7346376180648804,
+      "rewards/margins": 0.8208419680595398,
+      "rewards/rejected": -2.5554797649383545,
+      "step": 1600
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "eval_logits/chosen": -1.2524830102920532,
+      "eval_logits/rejected": -1.129266381263733,
+      "eval_logps/chosen": -434.1912536621094,
+      "eval_logps/rejected": -515.1350708007812,
+      "eval_loss": 0.49564477801322937,
+      "eval_rewards/accuracies": 0.7394999861717224,
+      "eval_rewards/chosen": -1.6958929300308228,
+      "eval_rewards/margins": 1.0097541809082031,
+      "eval_rewards/rejected": -2.7056469917297363,
+      "eval_runtime": 1597.9006,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 1600
+    },
+    {
+      "epoch": 0.4213556660560063,
+      "grad_norm": 5.6236395835876465,
+      "learning_rate": 3.586473211588787e-06,
+      "logits/chosen": -1.3308565616607666,
+      "logits/rejected": -1.2487056255340576,
+      "logps/chosen": -416.17218017578125,
+      "logps/rejected": -541.766357421875,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.7147547006607056,
+      "rewards/margins": 1.190341591835022,
+      "rewards/rejected": -2.9050965309143066,
+      "step": 1610
+    },
+    {
+      "epoch": 0.4239727819942423,
+      "grad_norm": 17.16832160949707,
+      "learning_rate": 3.5658536128085623e-06,
+      "logits/chosen": -1.3294860124588013,
+      "logits/rejected": -1.1633020639419556,
+      "logps/chosen": -514.1262817382812,
+      "logps/rejected": -586.5099487304688,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.5135364532470703,
+      "rewards/margins": 1.0195398330688477,
+      "rewards/rejected": -3.533076047897339,
+      "step": 1620
+    },
+    {
+      "epoch": 0.4265898979324784,
+      "grad_norm": 9.197494506835938,
+      "learning_rate": 3.545145015558399e-06,
+      "logits/chosen": -1.1497808694839478,
+      "logits/rejected": -1.1394188404083252,
+      "logps/chosen": -455.3041076660156,
+      "logps/rejected": -552.9832763671875,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.2378275394439697,
+      "rewards/margins": 1.1191097497940063,
+      "rewards/rejected": -3.3569374084472656,
+      "step": 1630
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 5.227996826171875,
+      "learning_rate": 3.5243491490002056e-06,
+      "logits/chosen": -1.344987154006958,
+      "logits/rejected": -1.2656126022338867,
+      "logps/chosen": -442.0404357910156,
+      "logps/rejected": -519.5951538085938,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8838779926300049,
+      "rewards/margins": 0.8285346031188965,
+      "rewards/rejected": -2.7124123573303223,
+      "step": 1640
+    },
+    {
+      "epoch": 0.4318241298089505,
+      "grad_norm": 6.703832149505615,
+      "learning_rate": 3.503467749582857e-06,
+      "logits/chosen": -1.4452259540557861,
+      "logits/rejected": -1.273822546005249,
+      "logps/chosen": -400.95703125,
+      "logps/rejected": -432.19775390625,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.4979441165924072,
+      "rewards/margins": 0.6527599096298218,
+      "rewards/rejected": -2.1507039070129395,
+      "step": 1650
+    },
+    {
+      "epoch": 0.4344412457471866,
+      "grad_norm": 11.32523250579834,
+      "learning_rate": 3.4825025608971947e-06,
+      "logits/chosen": -1.386530876159668,
+      "logits/rejected": -1.324684500694275,
+      "logps/chosen": -355.1617736816406,
+      "logps/rejected": -427.02001953125,
+      "loss": 0.5318,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3329139947891235,
+      "rewards/margins": 0.6805238723754883,
+      "rewards/rejected": -2.0134379863739014,
+      "step": 1660
+    },
+    {
+      "epoch": 0.43705836168542267,
+      "grad_norm": 7.073009490966797,
+      "learning_rate": 3.4614553335304407e-06,
+      "logits/chosen": -1.3409563302993774,
+      "logits/rejected": -1.1487702131271362,
+      "logps/chosen": -427.2156677246094,
+      "logps/rejected": -493.6348571777344,
+      "loss": 0.4728,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.584120273590088,
+      "rewards/margins": 0.9814236760139465,
+      "rewards/rejected": -2.5655438899993896,
+      "step": 1670
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 11.820454597473145,
+      "learning_rate": 3.4403278249200222e-06,
+      "logits/chosen": -1.262634038925171,
+      "logits/rejected": -1.0713304281234741,
+      "logps/chosen": -481.8815002441406,
+      "logps/rejected": -564.7189331054688,
+      "loss": 0.4399,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.9938242435455322,
+      "rewards/margins": 1.2474219799041748,
+      "rewards/rejected": -3.241246461868286,
+      "step": 1680
+    },
+    {
+      "epoch": 0.44229259356189476,
+      "grad_norm": 19.43402671813965,
+      "learning_rate": 3.4191217992068293e-06,
+      "logits/chosen": -1.2867403030395508,
+      "logits/rejected": -1.1040995121002197,
+      "logps/chosen": -548.404296875,
+      "logps/rejected": -614.572998046875,
+      "loss": 0.5467,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.703883647918701,
+      "rewards/margins": 1.1177372932434082,
+      "rewards/rejected": -3.821620464324951,
+      "step": 1690
+    },
+    {
+      "epoch": 0.44490970950013087,
+      "grad_norm": 18.481212615966797,
+      "learning_rate": 3.3978390270879056e-06,
+      "logits/chosen": -1.169626235961914,
+      "logits/rejected": -1.0800492763519287,
+      "logps/chosen": -525.806884765625,
+      "logps/rejected": -649.9021606445312,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.1594154834747314,
+      "rewards/margins": 1.1724364757537842,
+      "rewards/rejected": -4.331852436065674,
+      "step": 1700
+    },
+    {
+      "epoch": 0.44490970950013087,
+      "eval_logits/chosen": -1.1773556470870972,
+      "eval_logits/rejected": -1.0519527196884155,
+      "eval_logps/chosen": -568.2142333984375,
+      "eval_logps/rejected": -668.4868774414062,
+      "eval_loss": 0.5111355781555176,
+      "eval_rewards/accuracies": 0.7304999828338623,
+      "eval_rewards/chosen": -3.036123275756836,
+      "eval_rewards/margins": 1.203041672706604,
+      "eval_rewards/rejected": -4.23916482925415,
+      "eval_runtime": 1598.229,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 1700
+    },
+    {
+      "epoch": 0.4475268254383669,
+      "grad_norm": 17.149526596069336,
+      "learning_rate": 3.3764812856685995e-06,
+      "logits/chosen": -1.3090331554412842,
+      "logits/rejected": -1.292729139328003,
+      "logps/chosen": -502.16497802734375,
+      "logps/rejected": -622.8278198242188,
+      "loss": 0.5266,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.7771124839782715,
+      "rewards/margins": 1.0713131427764893,
+      "rewards/rejected": -3.8484256267547607,
+      "step": 1710
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 12.321800231933594,
+      "learning_rate": 3.3550503583141726e-06,
+      "logits/chosen": -1.5342642068862915,
+      "logits/rejected": -1.4036284685134888,
+      "logps/chosen": -507.0011291503906,
+      "logps/rejected": -602.5591430664062,
+      "loss": 0.497,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.3824760913848877,
+      "rewards/margins": 1.1078113317489624,
+      "rewards/rejected": -3.4902870655059814,
+      "step": 1720
+    },
+    {
+      "epoch": 0.45276105731483907,
+      "grad_norm": 7.4277191162109375,
+      "learning_rate": 3.3335480345008907e-06,
+      "logits/chosen": -1.5070991516113281,
+      "logits/rejected": -1.4292500019073486,
+      "logps/chosen": -427.82818603515625,
+      "logps/rejected": -509.09552001953125,
+      "loss": 0.4635,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.6776756048202515,
+      "rewards/margins": 1.0597718954086304,
+      "rewards/rejected": -2.737447500228882,
+      "step": 1730
+    },
+    {
+      "epoch": 0.4553781732530751,
+      "grad_norm": 9.2946195602417,
+      "learning_rate": 3.3119761096666055e-06,
+      "logits/chosen": -1.5444905757904053,
+      "logits/rejected": -1.4140485525131226,
+      "logps/chosen": -468.56182861328125,
+      "logps/rejected": -515.2850341796875,
+      "loss": 0.5735,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.9366995096206665,
+      "rewards/margins": 0.7699880599975586,
+      "rewards/rejected": -2.7066876888275146,
+      "step": 1740
+    },
+    {
+      "epoch": 0.45799528919131116,
+      "grad_norm": 7.504373073577881,
+      "learning_rate": 3.290336385060832e-06,
+      "logits/chosen": -1.724323034286499,
+      "logits/rejected": -1.5582685470581055,
+      "logps/chosen": -436.670166015625,
+      "logps/rejected": -503.16705322265625,
+      "loss": 0.5256,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8786998987197876,
+      "rewards/margins": 0.8662854433059692,
+      "rewards/rejected": -2.7449851036071777,
+      "step": 1750
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 11.6707181930542,
+      "learning_rate": 3.268630667594348e-06,
+      "logits/chosen": -1.552310824394226,
+      "logits/rejected": -1.5216782093048096,
+      "logps/chosen": -447.37835693359375,
+      "logps/rejected": -520.7014770507812,
+      "loss": 0.507,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9179880619049072,
+      "rewards/margins": 0.9757116436958313,
+      "rewards/rejected": -2.8936996459960938,
+      "step": 1760
+    },
+    {
+      "epoch": 0.4632295210677833,
+      "grad_norm": 10.357820510864258,
+      "learning_rate": 3.2468607696883147e-06,
+      "logits/chosen": -1.6096198558807373,
+      "logits/rejected": -1.5589017868041992,
+      "logps/chosen": -452.2684631347656,
+      "logps/rejected": -553.5985717773438,
+      "loss": 0.4902,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.9917962551116943,
+      "rewards/margins": 0.9700411558151245,
+      "rewards/rejected": -2.9618372917175293,
+      "step": 1770
+    },
+    {
+      "epoch": 0.46584663700601936,
+      "grad_norm": 8.833939552307129,
+      "learning_rate": 3.225028509122944e-06,
+      "logits/chosen": -1.7435325384140015,
+      "logits/rejected": -1.6519057750701904,
+      "logps/chosen": -414.03179931640625,
+      "logps/rejected": -483.228515625,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.7762641906738281,
+      "rewards/margins": 0.8044807314872742,
+      "rewards/rejected": -2.580744504928589,
+      "step": 1780
+    },
+    {
+      "epoch": 0.4684637529442554,
+      "grad_norm": 9.815378189086914,
+      "learning_rate": 3.2031357088857083e-06,
+      "logits/chosen": -1.6989177465438843,
+      "logits/rejected": -1.6307258605957031,
+      "logps/chosen": -467.5003967285156,
+      "logps/rejected": -558.3304443359375,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.923940896987915,
+      "rewards/margins": 0.9426455497741699,
+      "rewards/rejected": -2.866586446762085,
+      "step": 1790
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 10.215496063232422,
+      "learning_rate": 3.181184197019127e-06,
+      "logits/chosen": -1.4613163471221924,
+      "logits/rejected": -1.3699003458023071,
+      "logps/chosen": -425.10430908203125,
+      "logps/rejected": -576.7496948242188,
+      "loss": 0.4705,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.9493353366851807,
+      "rewards/margins": 1.254817247390747,
+      "rewards/rejected": -3.2041525840759277,
+      "step": 1800
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "eval_logits/chosen": -1.4508103132247925,
+      "eval_logits/rejected": -1.3478518724441528,
+      "eval_logps/chosen": -476.96630859375,
+      "eval_logps/rejected": -563.5121459960938,
+      "eval_loss": 0.49488988518714905,
+      "eval_rewards/accuracies": 0.7434999942779541,
+      "eval_rewards/chosen": -2.123643636703491,
+      "eval_rewards/margins": 1.0657742023468018,
+      "eval_rewards/rejected": -3.189417839050293,
+      "eval_runtime": 1598.3976,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 1800
+    },
+    {
+      "epoch": 0.47369798482072756,
+      "grad_norm": 19.882471084594727,
+      "learning_rate": 3.159175806468126e-06,
+      "logits/chosen": -1.4092731475830078,
+      "logits/rejected": -1.2594877481460571,
+      "logps/chosen": -483.1397399902344,
+      "logps/rejected": -564.5753173828125,
+      "loss": 0.4771,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.345776081085205,
+      "rewards/margins": 1.1159356832504272,
+      "rewards/rejected": -3.4617114067077637,
+      "step": 1810
+    },
+    {
+      "epoch": 0.4763151007589636,
+      "grad_norm": 13.595392227172852,
+      "learning_rate": 3.1371123749269804e-06,
+      "logits/chosen": -1.4161027669906616,
+      "logits/rejected": -1.3486690521240234,
+      "logps/chosen": -579.3997192382812,
+      "logps/rejected": -658.3812255859375,
+      "loss": 0.588,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.961474895477295,
+      "rewards/margins": 0.9501525163650513,
+      "rewards/rejected": -3.9116275310516357,
+      "step": 1820
+    },
+    {
+      "epoch": 0.4789322166971997,
+      "grad_norm": 16.07357406616211,
+      "learning_rate": 3.114995744685877e-06,
+      "logits/chosen": -1.3573070764541626,
+      "logits/rejected": -1.3251278400421143,
+      "logps/chosen": -537.996826171875,
+      "logps/rejected": -629.4193115234375,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.90761399269104,
+      "rewards/margins": 1.0439841747283936,
+      "rewards/rejected": -3.9515984058380127,
+      "step": 1830
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 6.550096035003662,
+      "learning_rate": 3.0928277624770743e-06,
+      "logits/chosen": -1.5519769191741943,
+      "logits/rejected": -1.4119082689285278,
+      "logps/chosen": -502.74102783203125,
+      "logps/rejected": -586.8773193359375,
+      "loss": 0.5118,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.1812124252319336,
+      "rewards/margins": 1.1252543926239014,
+      "rewards/rejected": -3.306466579437256,
+      "step": 1840
+    },
+    {
+      "epoch": 0.4841664485736718,
+      "grad_norm": 7.670591354370117,
+      "learning_rate": 3.070610279320708e-06,
+      "logits/chosen": -1.464906930923462,
+      "logits/rejected": -1.32064950466156,
+      "logps/chosen": -504.308349609375,
+      "logps/rejected": -599.1265869140625,
+      "loss": 0.4443,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.1736900806427,
+      "rewards/margins": 1.1892703771591187,
+      "rewards/rejected": -3.3629603385925293,
+      "step": 1850
+    },
+    {
+      "epoch": 0.48678356451190785,
+      "grad_norm": 9.109015464782715,
+      "learning_rate": 3.0483451503702264e-06,
+      "logits/chosen": -1.249629020690918,
+      "logits/rejected": -1.168210744857788,
+      "logps/chosen": -578.9190063476562,
+      "logps/rejected": -669.9890747070312,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.928832530975342,
+      "rewards/margins": 1.1118719577789307,
+      "rewards/rejected": -4.040704250335693,
+      "step": 1860
+    },
+    {
+      "epoch": 0.48940068045014395,
+      "grad_norm": 10.058290481567383,
+      "learning_rate": 3.0260342347574916e-06,
+      "logits/chosen": -1.2091599702835083,
+      "logits/rejected": -1.0555975437164307,
+      "logps/chosen": -534.0882568359375,
+      "logps/rejected": -660.2562255859375,
+      "loss": 0.4437,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.618621826171875,
+      "rewards/margins": 1.4526309967041016,
+      "rewards/rejected": -4.071252822875977,
+      "step": 1870
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 10.198968887329102,
+      "learning_rate": 3.0036793954375358e-06,
+      "logits/chosen": -1.303740382194519,
+      "logits/rejected": -1.155242681503296,
+      "logps/chosen": -507.42529296875,
+      "logps/rejected": -583.2550048828125,
+      "loss": 0.4895,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.374905824661255,
+      "rewards/margins": 1.1803052425384521,
+      "rewards/rejected": -3.555210828781128,
+      "step": 1880
+    },
+    {
+      "epoch": 0.49463491232661605,
+      "grad_norm": 10.858015060424805,
+      "learning_rate": 2.981282499033009e-06,
+      "logits/chosen": -1.3813010454177856,
+      "logits/rejected": -1.2690826654434204,
+      "logps/chosen": -486.0970153808594,
+      "logps/rejected": -569.85888671875,
+      "loss": 0.5031,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0988192558288574,
+      "rewards/margins": 1.0918813943862915,
+      "rewards/rejected": -3.1907010078430176,
+      "step": 1890
+    },
+    {
+      "epoch": 0.49725202826485215,
+      "grad_norm": 10.221612930297852,
+      "learning_rate": 2.9588454156783163e-06,
+      "logits/chosen": -1.4239078760147095,
+      "logits/rejected": -1.2654526233673096,
+      "logps/chosen": -474.64825439453125,
+      "logps/rejected": -587.3582763671875,
+      "loss": 0.4447,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9338430166244507,
+      "rewards/margins": 1.3446595668792725,
+      "rewards/rejected": -3.2785022258758545,
+      "step": 1900
+    },
+    {
+      "epoch": 0.49725202826485215,
+      "eval_logits/chosen": -1.2950754165649414,
+      "eval_logits/rejected": -1.1710810661315918,
+      "eval_logps/chosen": -468.10113525390625,
+      "eval_logps/rejected": -559.6229248046875,
+      "eval_loss": 0.49835336208343506,
+      "eval_rewards/accuracies": 0.7419999837875366,
+      "eval_rewards/chosen": -2.0349912643432617,
+      "eval_rewards/margins": 1.1155344247817993,
+      "eval_rewards/rejected": -3.1505255699157715,
+      "eval_runtime": 1598.3655,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 1900
+    },
+    {
+      "epoch": 0.4998691442030882,
+      "grad_norm": 12.752076148986816,
+      "learning_rate": 2.9363700188634597e-06,
+      "logits/chosen": -1.3872668743133545,
+      "logits/rejected": -1.265047311782837,
+      "logps/chosen": -489.17291259765625,
+      "logps/rejected": -553.0140991210938,
+      "loss": 0.5231,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.242345094680786,
+      "rewards/margins": 1.008666753768921,
+      "rewards/rejected": -3.251011610031128,
+      "step": 1910
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 10.04758071899414,
+      "learning_rate": 2.9138581852776053e-06,
+      "logits/chosen": -1.368890404701233,
+      "logits/rejected": -1.2567319869995117,
+      "logps/chosen": -493.54638671875,
+      "logps/rejected": -586.2965087890625,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.339406728744507,
+      "rewards/margins": 1.0563156604766846,
+      "rewards/rejected": -3.3957226276397705,
+      "step": 1920
+    },
+    {
+      "epoch": 0.5051033760795604,
+      "grad_norm": 9.104912757873535,
+      "learning_rate": 2.8913117946523805e-06,
+      "logits/chosen": -1.4171912670135498,
+      "logits/rejected": -1.2455161809921265,
+      "logps/chosen": -490.3606872558594,
+      "logps/rejected": -558.6239013671875,
+      "loss": 0.4792,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.2399258613586426,
+      "rewards/margins": 1.0637315511703491,
+      "rewards/rejected": -3.3036580085754395,
+      "step": 1930
+    },
+    {
+      "epoch": 0.5077204920177963,
+      "grad_norm": 11.299101829528809,
+      "learning_rate": 2.8687327296049126e-06,
+      "logits/chosen": -1.4033154249191284,
+      "logits/rejected": -1.307733178138733,
+      "logps/chosen": -491.1643981933594,
+      "logps/rejected": -589.1052856445312,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.32118558883667,
+      "rewards/margins": 1.0772285461425781,
+      "rewards/rejected": -3.398413896560669,
+      "step": 1940
+    },
+    {
+      "epoch": 0.5103376079560324,
+      "grad_norm": 18.843233108520508,
+      "learning_rate": 2.8461228754806376e-06,
+      "logits/chosen": -1.4243779182434082,
+      "logits/rejected": -1.2675760984420776,
+      "logps/chosen": -507.2261657714844,
+      "logps/rejected": -582.9666748046875,
+      "loss": 0.5307,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.3146495819091797,
+      "rewards/margins": 1.0160276889801025,
+      "rewards/rejected": -3.330677032470703,
+      "step": 1950
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 9.132381439208984,
+      "learning_rate": 2.823484120195865e-06,
+      "logits/chosen": -1.4892734289169312,
+      "logits/rejected": -1.310719609260559,
+      "logps/chosen": -494.809814453125,
+      "logps/rejected": -583.18798828125,
+      "loss": 0.4256,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.1708030700683594,
+      "rewards/margins": 1.2277300357818604,
+      "rewards/rejected": -3.398533582687378,
+      "step": 1960
+    },
+    {
+      "epoch": 0.5155718398325045,
+      "grad_norm": 13.682202339172363,
+      "learning_rate": 2.8008183540801486e-06,
+      "logits/chosen": -1.2465951442718506,
+      "logits/rejected": -1.113993525505066,
+      "logps/chosen": -583.8721313476562,
+      "logps/rejected": -649.005126953125,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.103384494781494,
+      "rewards/margins": 1.1054545640945435,
+      "rewards/rejected": -4.208839416503906,
+      "step": 1970
+    },
+    {
+      "epoch": 0.5181889557707406,
+      "grad_norm": 13.151285171508789,
+      "learning_rate": 2.7781274697184353e-06,
+      "logits/chosen": -1.1965068578720093,
+      "logits/rejected": -1.2104531526565552,
+      "logps/chosen": -526.9979248046875,
+      "logps/rejected": -667.6734619140625,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.9773271083831787,
+      "rewards/margins": 1.2037287950515747,
+      "rewards/rejected": -4.181056022644043,
+      "step": 1980
+    },
+    {
+      "epoch": 0.5208060717089767,
+      "grad_norm": 7.820137023925781,
+      "learning_rate": 2.7554133617930397e-06,
+      "logits/chosen": -1.4265800714492798,
+      "logits/rejected": -1.3070918321609497,
+      "logps/chosen": -457.6055603027344,
+      "logps/rejected": -541.1328735351562,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.098328113555908,
+      "rewards/margins": 0.9863009452819824,
+      "rewards/rejected": -3.0846290588378906,
+      "step": 1990
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 12.656340599060059,
+      "learning_rate": 2.7326779269254363e-06,
+      "logits/chosen": -1.6775627136230469,
+      "logits/rejected": -1.5273171663284302,
+      "logps/chosen": -465.67138671875,
+      "logps/rejected": -513.0640869140625,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7640680074691772,
+      "rewards/margins": 1.0338157415390015,
+      "rewards/rejected": -2.797883987426758,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "eval_logits/chosen": -1.4695755243301392,
+      "eval_logits/rejected": -1.3557151556015015,
+      "eval_logps/chosen": -461.283935546875,
+      "eval_logps/rejected": -540.4462280273438,
+      "eval_loss": 0.4929336607456207,
+      "eval_rewards/accuracies": 0.7419999837875366,
+      "eval_rewards/chosen": -1.9668195247650146,
+      "eval_rewards/margins": 0.9919391870498657,
+      "eval_rewards/rejected": -2.95875883102417,
+      "eval_runtime": 1598.5788,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5260403035854488,
+      "grad_norm": 10.136625289916992,
+      "learning_rate": 2.7099230635178954e-06,
+      "logits/chosen": -1.506194829940796,
+      "logits/rejected": -1.4691191911697388,
+      "logps/chosen": -475.35992431640625,
+      "logps/rejected": -562.2814331054688,
+      "loss": 0.5025,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.106142520904541,
+      "rewards/margins": 0.9343917965888977,
+      "rewards/rejected": -3.040534257888794,
+      "step": 2010
+    },
+    {
+      "epoch": 0.528657419523685,
+      "grad_norm": 10.457864761352539,
+      "learning_rate": 2.6871506715949608e-06,
+      "logits/chosen": -1.5337005853652954,
+      "logits/rejected": -1.3907279968261719,
+      "logps/chosen": -485.59637451171875,
+      "logps/rejected": -580.8443603515625,
+      "loss": 0.4664,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.3269832134246826,
+      "rewards/margins": 1.1330835819244385,
+      "rewards/rejected": -3.4600670337677,
+      "step": 2020
+    },
+    {
+      "epoch": 0.5312745354619209,
+      "grad_norm": 7.218365669250488,
+      "learning_rate": 2.6643626526448063e-06,
+      "logits/chosen": -1.4807672500610352,
+      "logits/rejected": -1.308292031288147,
+      "logps/chosen": -573.4922485351562,
+      "logps/rejected": -676.8171997070312,
+      "loss": 0.4288,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.787353038787842,
+      "rewards/margins": 1.456699252128601,
+      "rewards/rejected": -4.244051933288574,
+      "step": 2030
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 14.370759010314941,
+      "learning_rate": 2.6415609094604562e-06,
+      "logits/chosen": -1.194697380065918,
+      "logits/rejected": -1.1315624713897705,
+      "logps/chosen": -666.68896484375,
+      "logps/rejected": -775.71240234375,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.9625601768493652,
+      "rewards/margins": 1.2808544635772705,
+      "rewards/rejected": -5.243414878845215,
+      "step": 2040
+    },
+    {
+      "epoch": 0.5365087673383931,
+      "grad_norm": 12.651047706604004,
+      "learning_rate": 2.618747345980904e-06,
+      "logits/chosen": -1.1674007177352905,
+      "logits/rejected": -0.9596608877182007,
+      "logps/chosen": -729.123291015625,
+      "logps/rejected": -793.8466796875,
+      "loss": 0.547,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.807937145233154,
+      "rewards/margins": 1.1525070667266846,
+      "rewards/rejected": -5.960444450378418,
+      "step": 2050
+    },
+    {
+      "epoch": 0.5391258832766291,
+      "grad_norm": 6.089327335357666,
+      "learning_rate": 2.595923867132136e-06,
+      "logits/chosen": -1.2956593036651611,
+      "logits/rejected": -1.1621012687683105,
+      "logps/chosen": -649.8438720703125,
+      "logps/rejected": -758.2276000976562,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.689380645751953,
+      "rewards/margins": 1.3314837217330933,
+      "rewards/rejected": -5.020864009857178,
+      "step": 2060
+    },
+    {
+      "epoch": 0.5417429992148652,
+      "grad_norm": 7.40191650390625,
+      "learning_rate": 2.5730923786680672e-06,
+      "logits/chosen": -1.2692714929580688,
+      "logits/rejected": -1.248396635055542,
+      "logps/chosen": -549.1704711914062,
+      "logps/rejected": -643.1365966796875,
+      "loss": 0.5449,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.976423740386963,
+      "rewards/margins": 0.864033579826355,
+      "rewards/rejected": -3.8404572010040283,
+      "step": 2070
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 9.540526390075684,
+      "learning_rate": 2.5502547870114137e-06,
+      "logits/chosen": -1.3864257335662842,
+      "logits/rejected": -1.2374814748764038,
+      "logps/chosen": -515.8956909179688,
+      "logps/rejected": -582.565185546875,
+      "loss": 0.5054,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.6055984497070312,
+      "rewards/margins": 0.9696043729782104,
+      "rewards/rejected": -3.5752029418945312,
+      "step": 2080
+    },
+    {
+      "epoch": 0.5469772310913373,
+      "grad_norm": 11.426345825195312,
+      "learning_rate": 2.527412999094507e-06,
+      "logits/chosen": -1.3803369998931885,
+      "logits/rejected": -1.2163236141204834,
+      "logps/chosen": -550.672607421875,
+      "logps/rejected": -663.0771484375,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.5704407691955566,
+      "rewards/margins": 1.2492586374282837,
+      "rewards/rejected": -3.81969952583313,
+      "step": 2090
+    },
+    {
+      "epoch": 0.5495943470295734,
+      "grad_norm": 10.400588035583496,
+      "learning_rate": 2.504568922200064e-06,
+      "logits/chosen": -1.3159221410751343,
+      "logits/rejected": -1.1753368377685547,
+      "logps/chosen": -523.4752197265625,
+      "logps/rejected": -638.3160400390625,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.8226757049560547,
+      "rewards/margins": 1.3093383312225342,
+      "rewards/rejected": -4.132014274597168,
+      "step": 2100
+    },
+    {
+      "epoch": 0.5495943470295734,
+      "eval_logits/chosen": -1.2426347732543945,
+      "eval_logits/rejected": -1.1150033473968506,
+      "eval_logps/chosen": -579.1231079101562,
+      "eval_logps/rejected": -680.8953857421875,
+      "eval_loss": 0.49687275290489197,
+      "eval_rewards/accuracies": 0.7350000143051147,
+      "eval_rewards/chosen": -3.145211935043335,
+      "eval_rewards/margins": 1.2180382013320923,
+      "eval_rewards/rejected": -4.363250255584717,
+      "eval_runtime": 1598.2336,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 2100
+    },
+    {
+      "epoch": 0.5522114629678094,
+      "grad_norm": 13.339751243591309,
+      "learning_rate": 2.4817244638019333e-06,
+      "logits/chosen": -1.3830143213272095,
+      "logits/rejected": -1.2277696132659912,
+      "logps/chosen": -579.9473876953125,
+      "logps/rejected": -659.97412109375,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.041943073272705,
+      "rewards/margins": 1.2124286890029907,
+      "rewards/rejected": -4.254371643066406,
+      "step": 2110
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 10.70681095123291,
+      "learning_rate": 2.4588815314058155e-06,
+      "logits/chosen": -1.3805171251296997,
+      "logits/rejected": -1.3095520734786987,
+      "logps/chosen": -527.185302734375,
+      "logps/rejected": -602.2315673828125,
+      "loss": 0.4666,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.8171143531799316,
+      "rewards/margins": 1.0927484035491943,
+      "rewards/rejected": -3.909862518310547,
+      "step": 2120
+    },
+    {
+      "epoch": 0.5574456948442816,
+      "grad_norm": 13.302124977111816,
+      "learning_rate": 2.4360420323899922e-06,
+      "logits/chosen": -1.4553115367889404,
+      "logits/rejected": -1.3466088771820068,
+      "logps/chosen": -556.26611328125,
+      "logps/rejected": -608.5457763671875,
+      "loss": 0.5919,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.8050436973571777,
+      "rewards/margins": 0.8486245274543762,
+      "rewards/rejected": -3.653668165206909,
+      "step": 2130
+    },
+    {
+      "epoch": 0.5600628107825176,
+      "grad_norm": 9.734771728515625,
+      "learning_rate": 2.4132078738460585e-06,
+      "logits/chosen": -1.5308691263198853,
+      "logits/rejected": -1.3970110416412354,
+      "logps/chosen": -516.7899169921875,
+      "logps/rejected": -586.8917236328125,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.5241827964782715,
+      "rewards/margins": 1.0863929986953735,
+      "rewards/rejected": -3.6105754375457764,
+      "step": 2140
+    },
+    {
+      "epoch": 0.5626799267207537,
+      "grad_norm": 16.153608322143555,
+      "learning_rate": 2.3903809624196826e-06,
+      "logits/chosen": -1.4662330150604248,
+      "logits/rejected": -1.3525559902191162,
+      "logps/chosen": -481.68035888671875,
+      "logps/rejected": -539.03515625,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.4799540042877197,
+      "rewards/margins": 0.8992452621459961,
+      "rewards/rejected": -3.379199266433716,
+      "step": 2150
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 13.256193161010742,
+      "learning_rate": 2.3675632041513978e-06,
+      "logits/chosen": -1.5737719535827637,
+      "logits/rejected": -1.3470098972320557,
+      "logps/chosen": -527.2965698242188,
+      "logps/rejected": -575.4271240234375,
+      "loss": 0.4882,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.403984546661377,
+      "rewards/margins": 1.1281957626342773,
+      "rewards/rejected": -3.532180070877075,
+      "step": 2160
+    },
+    {
+      "epoch": 0.5679141585972258,
+      "grad_norm": 13.135820388793945,
+      "learning_rate": 2.3447565043174533e-06,
+      "logits/chosen": -1.4236756563186646,
+      "logits/rejected": -1.263068437576294,
+      "logps/chosen": -507.3004455566406,
+      "logps/rejected": -562.6768798828125,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.490262746810913,
+      "rewards/margins": 0.9420592188835144,
+      "rewards/rejected": -3.4323222637176514,
+      "step": 2170
+    },
+    {
+      "epoch": 0.5705312745354619,
+      "grad_norm": 11.776784896850586,
+      "learning_rate": 2.321962767270724e-06,
+      "logits/chosen": -1.4430488348007202,
+      "logits/rejected": -1.3176742792129517,
+      "logps/chosen": -504.53009033203125,
+      "logps/rejected": -551.927490234375,
+      "loss": 0.5579,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.546835422515869,
+      "rewards/margins": 0.8456700444221497,
+      "rewards/rejected": -3.392504930496216,
+      "step": 2180
+    },
+    {
+      "epoch": 0.573148390473698,
+      "grad_norm": 8.283063888549805,
+      "learning_rate": 2.299183896281692e-06,
+      "logits/chosen": -1.4085118770599365,
+      "logits/rejected": -1.3014566898345947,
+      "logps/chosen": -492.5560607910156,
+      "logps/rejected": -586.7115478515625,
+      "loss": 0.5087,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.376844882965088,
+      "rewards/margins": 0.9645326733589172,
+      "rewards/rejected": -3.3413777351379395,
+      "step": 2190
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 6.720417022705078,
+      "learning_rate": 2.2764217933795297e-06,
+      "logits/chosen": -1.529714822769165,
+      "logits/rejected": -1.415728211402893,
+      "logps/chosen": -480.97955322265625,
+      "logps/rejected": -568.6060791015625,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.196598529815674,
+      "rewards/margins": 1.081301212310791,
+      "rewards/rejected": -3.2778995037078857,
+      "step": 2200
+    },
+    {
+      "epoch": 0.575765506411934,
+      "eval_logits/chosen": -1.3886340856552124,
+      "eval_logits/rejected": -1.2705532312393188,
+      "eval_logps/chosen": -502.5681457519531,
+      "eval_logps/rejected": -588.3314819335938,
+      "eval_loss": 0.49266016483306885,
+      "eval_rewards/accuracies": 0.7404999732971191,
+      "eval_rewards/chosen": -2.379661798477173,
+      "eval_rewards/margins": 1.0579497814178467,
+      "eval_rewards/rejected": -3.4376115798950195,
+      "eval_runtime": 1597.9898,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 2200
+    },
+    {
+      "epoch": 0.5783826223501701,
+      "grad_norm": 5.171483039855957,
+      "learning_rate": 2.2536783591932786e-06,
+      "logits/chosen": -1.5503586530685425,
+      "logits/rejected": -1.394595742225647,
+      "logps/chosen": -533.165771484375,
+      "logps/rejected": -615.0496826171875,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.578037738800049,
+      "rewards/margins": 1.0000585317611694,
+      "rewards/rejected": -3.5780959129333496,
+      "step": 2210
+    },
+    {
+      "epoch": 0.5809997382884062,
+      "grad_norm": 8.2252836227417,
+      "learning_rate": 2.230955492793149e-06,
+      "logits/chosen": -1.3457584381103516,
+      "logits/rejected": -1.2957208156585693,
+      "logps/chosen": -549.8411865234375,
+      "logps/rejected": -641.9370727539062,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.760831356048584,
+      "rewards/margins": 1.0501973628997803,
+      "rewards/rejected": -3.811028242111206,
+      "step": 2220
+    },
+    {
+      "epoch": 0.5836168542266422,
+      "grad_norm": 8.77107048034668,
+      "learning_rate": 2.208255091531947e-06,
+      "logits/chosen": -1.3714348077774048,
+      "logits/rejected": -1.294524908065796,
+      "logps/chosen": -516.0252685546875,
+      "logps/rejected": -604.4722900390625,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.39119291305542,
+      "rewards/margins": 1.2340974807739258,
+      "rewards/rejected": -3.6252903938293457,
+      "step": 2230
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 13.218793869018555,
+      "learning_rate": 2.1855790508866435e-06,
+      "logits/chosen": -1.4534399509429932,
+      "logits/rejected": -1.355835199356079,
+      "logps/chosen": -526.3766479492188,
+      "logps/rejected": -626.1098022460938,
+      "loss": 0.5044,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.2947020530700684,
+      "rewards/margins": 1.1731860637664795,
+      "rewards/rejected": -3.467888355255127,
+      "step": 2240
+    },
+    {
+      "epoch": 0.5888510861031143,
+      "grad_norm": 7.438356876373291,
+      "learning_rate": 2.162929264300107e-06,
+      "logits/chosen": -1.4775898456573486,
+      "logits/rejected": -1.375610113143921,
+      "logps/chosen": -440.20208740234375,
+      "logps/rejected": -556.2523193359375,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.712977647781372,
+      "rewards/margins": 1.3840254545211792,
+      "rewards/rejected": -3.0970029830932617,
+      "step": 2250
+    },
+    {
+      "epoch": 0.5914682020413504,
+      "grad_norm": 13.876564025878906,
+      "learning_rate": 2.1403076230230006e-06,
+      "logits/chosen": -1.4080345630645752,
+      "logits/rejected": -1.2920736074447632,
+      "logps/chosen": -475.05548095703125,
+      "logps/rejected": -532.099365234375,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.039393901824951,
+      "rewards/margins": 0.848638653755188,
+      "rewards/rejected": -2.8880326747894287,
+      "step": 2260
+    },
+    {
+      "epoch": 0.5940853179795865,
+      "grad_norm": 10.589466094970703,
+      "learning_rate": 2.11771601595586e-06,
+      "logits/chosen": -1.447265386581421,
+      "logits/rejected": -1.3221790790557861,
+      "logps/chosen": -478.97149658203125,
+      "logps/rejected": -525.7181396484375,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.9702937602996826,
+      "rewards/margins": 0.9898223876953125,
+      "rewards/rejected": -2.960115909576416,
+      "step": 2270
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 13.875945091247559,
+      "learning_rate": 2.0951563294913737e-06,
+      "logits/chosen": -1.3917208909988403,
+      "logits/rejected": -1.2089694738388062,
+      "logps/chosen": -480.66595458984375,
+      "logps/rejected": -556.7701416015625,
+      "loss": 0.462,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.19204044342041,
+      "rewards/margins": 1.0662552118301392,
+      "rewards/rejected": -3.2582955360412598,
+      "step": 2280
+    },
+    {
+      "epoch": 0.5993195498560586,
+      "grad_norm": 9.446721076965332,
+      "learning_rate": 2.0726304473568693e-06,
+      "logits/chosen": -1.3553434610366821,
+      "logits/rejected": -1.238797903060913,
+      "logps/chosen": -493.4701232910156,
+      "logps/rejected": -565.8324584960938,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.3467798233032227,
+      "rewards/margins": 1.046331524848938,
+      "rewards/rejected": -3.393110990524292,
+      "step": 2290
+    },
+    {
+      "epoch": 0.6019366657942947,
+      "grad_norm": 9.440372467041016,
+      "learning_rate": 2.050140250457023e-06,
+      "logits/chosen": -1.4484026432037354,
+      "logits/rejected": -1.2413018941879272,
+      "logps/chosen": -548.2637939453125,
+      "logps/rejected": -634.7371826171875,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.7083754539489746,
+      "rewards/margins": 1.2052128314971924,
+      "rewards/rejected": -3.913588285446167,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6019366657942947,
+      "eval_logits/chosen": -1.2145209312438965,
+      "eval_logits/rejected": -1.0868196487426758,
+      "eval_logps/chosen": -549.21240234375,
+      "eval_logps/rejected": -656.666748046875,
+      "eval_loss": 0.49239280819892883,
+      "eval_rewards/accuracies": 0.7404999732971191,
+      "eval_rewards/chosen": -2.846104383468628,
+      "eval_rewards/margins": 1.2748597860336304,
+      "eval_rewards/rejected": -4.120964050292969,
+      "eval_runtime": 1598.0392,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6045537817325307,
+      "grad_norm": 15.882525444030762,
+      "learning_rate": 2.0276876167168042e-06,
+      "logits/chosen": -1.1735626459121704,
+      "logits/rejected": -1.07330322265625,
+      "logps/chosen": -511.028564453125,
+      "logps/rejected": -598.4305419921875,
+      "loss": 0.5852,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.8660171031951904,
+      "rewards/margins": 1.2002947330474854,
+      "rewards/rejected": -4.066311836242676,
+      "step": 2310
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 8.326827049255371,
+      "learning_rate": 2.0052744209246682e-06,
+      "logits/chosen": -1.3162415027618408,
+      "logits/rejected": -1.1979725360870361,
+      "logps/chosen": -522.6014404296875,
+      "logps/rejected": -590.3125610351562,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.673841714859009,
+      "rewards/margins": 1.005782127380371,
+      "rewards/rejected": -3.6796233654022217,
+      "step": 2320
+    },
+    {
+      "epoch": 0.6097880136090029,
+      "grad_norm": 14.694316864013672,
+      "learning_rate": 1.9829025345760127e-06,
+      "logits/chosen": -1.321624994277954,
+      "logits/rejected": -1.2669956684112549,
+      "logps/chosen": -527.9610595703125,
+      "logps/rejected": -604.3615112304688,
+      "loss": 0.5587,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.490408420562744,
+      "rewards/margins": 0.8247787356376648,
+      "rewards/rejected": -3.315187454223633,
+      "step": 2330
+    },
+    {
+      "epoch": 0.6124051295472389,
+      "grad_norm": 10.311464309692383,
+      "learning_rate": 1.9605738257169115e-06,
+      "logits/chosen": -1.270620346069336,
+      "logits/rejected": -1.120276927947998,
+      "logps/chosen": -485.13226318359375,
+      "logps/rejected": -588.9454956054688,
+      "loss": 0.4972,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.5224430561065674,
+      "rewards/margins": 1.1178371906280518,
+      "rewards/rejected": -3.6402804851531982,
+      "step": 2340
+    },
+    {
+      "epoch": 0.615022245485475,
+      "grad_norm": 10.49399471282959,
+      "learning_rate": 1.9382901587881275e-06,
+      "logits/chosen": -1.304638147354126,
+      "logits/rejected": -1.1775546073913574,
+      "logps/chosen": -500.91339111328125,
+      "logps/rejected": -595.765625,
+      "loss": 0.4158,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.42271089553833,
+      "rewards/margins": 1.271071195602417,
+      "rewards/rejected": -3.693782329559326,
+      "step": 2350
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 10.7622652053833,
+      "learning_rate": 1.916053394469437e-06,
+      "logits/chosen": -1.3254437446594238,
+      "logits/rejected": -1.1220028400421143,
+      "logps/chosen": -544.5164184570312,
+      "logps/rejected": -650.4244384765625,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.812866687774658,
+      "rewards/margins": 1.2050769329071045,
+      "rewards/rejected": -4.017943382263184,
+      "step": 2360
+    },
+    {
+      "epoch": 0.6202564773619471,
+      "grad_norm": 9.485381126403809,
+      "learning_rate": 1.8938653895242604e-06,
+      "logits/chosen": -1.2691552639007568,
+      "logits/rejected": -1.0826399326324463,
+      "logps/chosen": -547.7079467773438,
+      "logps/rejected": -651.6534423828125,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.8349509239196777,
+      "rewards/margins": 1.3276488780975342,
+      "rewards/rejected": -4.162599563598633,
+      "step": 2370
+    },
+    {
+      "epoch": 0.6228735933001832,
+      "grad_norm": 13.950716018676758,
+      "learning_rate": 1.8717279966446267e-06,
+      "logits/chosen": -1.116172432899475,
+      "logits/rejected": -1.0301882028579712,
+      "logps/chosen": -559.43896484375,
+      "logps/rejected": -674.9713134765625,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.109766721725464,
+      "rewards/margins": 1.2503349781036377,
+      "rewards/rejected": -4.360101699829102,
+      "step": 2380
+    },
+    {
+      "epoch": 0.6254907092384192,
+      "grad_norm": 6.874395370483398,
+      "learning_rate": 1.8496430642964698e-06,
+      "logits/chosen": -1.1902521848678589,
+      "logits/rejected": -1.0676952600479126,
+      "logps/chosen": -581.3973999023438,
+      "logps/rejected": -685.8600463867188,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.091068983078003,
+      "rewards/margins": 1.280792474746704,
+      "rewards/rejected": -4.371861457824707,
+      "step": 2390
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 8.19883918762207,
+      "learning_rate": 1.827612436565286e-06,
+      "logits/chosen": -1.2159771919250488,
+      "logits/rejected": -1.0594358444213867,
+      "logps/chosen": -552.8912353515625,
+      "logps/rejected": -668.69921875,
+      "loss": 0.4501,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.900498390197754,
+      "rewards/margins": 1.363520622253418,
+      "rewards/rejected": -4.264018535614014,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "eval_logits/chosen": -1.125669240951538,
+      "eval_logits/rejected": -0.9978408813476562,
+      "eval_logps/chosen": -562.0332641601562,
+      "eval_logps/rejected": -668.2345581054688,
+      "eval_loss": 0.489955335855484,
+      "eval_rewards/accuracies": 0.7429999709129333,
+      "eval_rewards/chosen": -2.974313735961914,
+      "eval_rewards/margins": 1.2623279094696045,
+      "eval_rewards/rejected": -4.236640930175781,
+      "eval_runtime": 1597.7712,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6307249411148914,
+      "grad_norm": 17.752099990844727,
+      "learning_rate": 1.8056379530021492e-06,
+      "logits/chosen": -1.2881437540054321,
+      "logits/rejected": -1.1982684135437012,
+      "logps/chosen": -541.344482421875,
+      "logps/rejected": -619.2030029296875,
+      "loss": 0.5127,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.9722681045532227,
+      "rewards/margins": 1.0265535116195679,
+      "rewards/rejected": -3.998821258544922,
+      "step": 2410
+    },
+    {
+      "epoch": 0.6333420570531274,
+      "grad_norm": 8.487099647521973,
+      "learning_rate": 1.7837214484701154e-06,
+      "logits/chosen": -1.339787244796753,
+      "logits/rejected": -1.2054253816604614,
+      "logps/chosen": -510.6474609375,
+      "logps/rejected": -616.0956420898438,
+      "loss": 0.4786,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.5842533111572266,
+      "rewards/margins": 1.2878291606903076,
+      "rewards/rejected": -3.872082233428955,
+      "step": 2420
+    },
+    {
+      "epoch": 0.6359591729913635,
+      "grad_norm": 14.060596466064453,
+      "learning_rate": 1.7618647529910043e-06,
+      "logits/chosen": -1.3402836322784424,
+      "logits/rejected": -1.2104113101959229,
+      "logps/chosen": -508.7401428222656,
+      "logps/rejected": -625.2578735351562,
+      "loss": 0.4786,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.520415782928467,
+      "rewards/margins": 1.2837390899658203,
+      "rewards/rejected": -3.804154872894287,
+      "step": 2430
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 9.59952449798584,
+      "learning_rate": 1.7400696915925996e-06,
+      "logits/chosen": -1.3208459615707397,
+      "logits/rejected": -1.141103982925415,
+      "logps/chosen": -542.9054565429688,
+      "logps/rejected": -597.6497802734375,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.7234339714050293,
+      "rewards/margins": 1.1302305459976196,
+      "rewards/rejected": -3.853663921356201,
+      "step": 2440
+    },
+    {
+      "epoch": 0.6411934048678356,
+      "grad_norm": 12.0162992477417,
+      "learning_rate": 1.718338084156254e-06,
+      "logits/chosen": -1.309754490852356,
+      "logits/rejected": -1.1585099697113037,
+      "logps/chosen": -527.2728881835938,
+      "logps/rejected": -612.1050415039062,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.4056506156921387,
+      "rewards/margins": 1.2328300476074219,
+      "rewards/rejected": -3.6384806632995605,
+      "step": 2450
+    },
+    {
+      "epoch": 0.6438105208060717,
+      "grad_norm": 15.278785705566406,
+      "learning_rate": 1.6966717452649372e-06,
+      "logits/chosen": -1.446975588798523,
+      "logits/rejected": -1.282833456993103,
+      "logps/chosen": -502.6387634277344,
+      "logps/rejected": -571.8133544921875,
+      "loss": 0.4497,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2578883171081543,
+      "rewards/margins": 1.2107127904891968,
+      "rewards/rejected": -3.4686012268066406,
+      "step": 2460
+    },
+    {
+      "epoch": 0.6464276367443078,
+      "grad_norm": 8.051189422607422,
+      "learning_rate": 1.6750724840517103e-06,
+      "logits/chosen": -1.3973640203475952,
+      "logits/rejected": -1.3273109197616577,
+      "logps/chosen": -461.38665771484375,
+      "logps/rejected": -567.72509765625,
+      "loss": 0.5096,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.086717128753662,
+      "rewards/margins": 1.0224206447601318,
+      "rewards/rejected": -3.109138011932373,
+      "step": 2470
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 11.723711967468262,
+      "learning_rate": 1.6535421040486686e-06,
+      "logits/chosen": -1.2049771547317505,
+      "logits/rejected": -1.1100887060165405,
+      "logps/chosen": -472.3949279785156,
+      "logps/rejected": -566.0884399414062,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.191418409347534,
+      "rewards/margins": 1.2799413204193115,
+      "rewards/rejected": -3.4713597297668457,
+      "step": 2480
+    },
+    {
+      "epoch": 0.6516618686207799,
+      "grad_norm": 18.185462951660156,
+      "learning_rate": 1.6320824030363458e-06,
+      "logits/chosen": -1.2619669437408447,
+      "logits/rejected": -1.2079191207885742,
+      "logps/chosen": -466.91845703125,
+      "logps/rejected": -579.8604125976562,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.3208956718444824,
+      "rewards/margins": 1.2844674587249756,
+      "rewards/rejected": -3.605363130569458,
+      "step": 2490
+    },
+    {
+      "epoch": 0.654278984559016,
+      "grad_norm": 14.226390838623047,
+      "learning_rate": 1.6106951728936028e-06,
+      "logits/chosen": -1.3449714183807373,
+      "logits/rejected": -1.212172508239746,
+      "logps/chosen": -500.9017028808594,
+      "logps/rejected": -603.9783325195312,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.4372124671936035,
+      "rewards/margins": 1.1106388568878174,
+      "rewards/rejected": -3.547851085662842,
+      "step": 2500
+    },
+    {
+      "epoch": 0.654278984559016,
+      "eval_logits/chosen": -1.1862049102783203,
+      "eval_logits/rejected": -1.0531891584396362,
+      "eval_logps/chosen": -510.45111083984375,
+      "eval_logps/rejected": -612.1486206054688,
+      "eval_loss": 0.4872073829174042,
+      "eval_rewards/accuracies": 0.7419999837875366,
+      "eval_rewards/chosen": -2.4584920406341553,
+      "eval_rewards/margins": 1.2172898054122925,
+      "eval_rewards/rejected": -3.675781488418579,
+      "eval_runtime": 1597.3321,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 2500
+    },
+    {
+      "epoch": 0.656896100497252,
+      "grad_norm": 10.693954467773438,
+      "learning_rate": 1.5893821994479996e-06,
+      "logits/chosen": -1.3317979574203491,
+      "logits/rejected": -1.2107937335968018,
+      "logps/chosen": -518.4515380859375,
+      "logps/rejected": -599.1994018554688,
+      "loss": 0.479,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.410822629928589,
+      "rewards/margins": 1.1930694580078125,
+      "rewards/rejected": -3.6038920879364014,
+      "step": 2510
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 10.660543441772461,
+      "learning_rate": 1.5681452623266868e-06,
+      "logits/chosen": -1.3140581846237183,
+      "logits/rejected": -1.059616208076477,
+      "logps/chosen": -549.7576293945312,
+      "logps/rejected": -629.7240600585938,
+      "loss": 0.4838,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.60799503326416,
+      "rewards/margins": 1.3822691440582275,
+      "rewards/rejected": -3.9902641773223877,
+      "step": 2520
+    },
+    {
+      "epoch": 0.6621303323737242,
+      "grad_norm": 9.327082633972168,
+      "learning_rate": 1.5469861348078014e-06,
+      "logits/chosen": -1.315731167793274,
+      "logits/rejected": -1.1592333316802979,
+      "logps/chosen": -516.8220825195312,
+      "logps/rejected": -651.5704956054688,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.7009530067443848,
+      "rewards/margins": 1.4539804458618164,
+      "rewards/rejected": -4.154933929443359,
+      "step": 2530
+    },
+    {
+      "epoch": 0.6647474483119602,
+      "grad_norm": 14.643211364746094,
+      "learning_rate": 1.5259065836724035e-06,
+      "logits/chosen": -1.1933174133300781,
+      "logits/rejected": -1.119011402130127,
+      "logps/chosen": -520.9530639648438,
+      "logps/rejected": -660.1638793945312,
+      "loss": 0.4109,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.762756824493408,
+      "rewards/margins": 1.4529297351837158,
+      "rewards/rejected": -4.215685844421387,
+      "step": 2540
+    },
+    {
+      "epoch": 0.6673645642501963,
+      "grad_norm": 23.931337356567383,
+      "learning_rate": 1.5049083690569456e-06,
+      "logits/chosen": -1.2723230123519897,
+      "logits/rejected": -1.1774482727050781,
+      "logps/chosen": -514.2723999023438,
+      "logps/rejected": -640.9010009765625,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.768399477005005,
+      "rewards/margins": 1.3103950023651123,
+      "rewards/rejected": -4.078794002532959,
+      "step": 2550
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 10.859006881713867,
+      "learning_rate": 1.4839932443063057e-06,
+      "logits/chosen": -1.286709189414978,
+      "logits/rejected": -1.130676507949829,
+      "logps/chosen": -553.838134765625,
+      "logps/rejected": -631.344482421875,
+      "loss": 0.459,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.619520425796509,
+      "rewards/margins": 1.3386470079421997,
+      "rewards/rejected": -3.958167314529419,
+      "step": 2560
+    },
+    {
+      "epoch": 0.6725987961266684,
+      "grad_norm": 14.106218338012695,
+      "learning_rate": 1.4631629558273803e-06,
+      "logits/chosen": -1.3286449909210205,
+      "logits/rejected": -1.205742597579956,
+      "logps/chosen": -493.1719665527344,
+      "logps/rejected": -586.0244140625,
+      "loss": 0.5866,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.515252113342285,
+      "rewards/margins": 1.044698715209961,
+      "rewards/rejected": -3.559950590133667,
+      "step": 2570
+    },
+    {
+      "epoch": 0.6752159120649045,
+      "grad_norm": 6.197361469268799,
+      "learning_rate": 1.4424192429432657e-06,
+      "logits/chosen": -1.4131492376327515,
+      "logits/rejected": -1.3109676837921143,
+      "logps/chosen": -469.304931640625,
+      "logps/rejected": -598.4210815429688,
+      "loss": 0.4614,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.13527250289917,
+      "rewards/margins": 1.2581441402435303,
+      "rewards/rejected": -3.3934166431427,
+      "step": 2580
+    },
+    {
+      "epoch": 0.6778330280031405,
+      "grad_norm": 13.008953094482422,
+      "learning_rate": 1.421763837748016e-06,
+      "logits/chosen": -1.3615756034851074,
+      "logits/rejected": -1.268027901649475,
+      "logps/chosen": -481.23358154296875,
+      "logps/rejected": -605.3997802734375,
+      "loss": 0.4241,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.3075268268585205,
+      "rewards/margins": 1.3260997533798218,
+      "rewards/rejected": -3.6336264610290527,
+      "step": 2590
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 10.1073637008667,
+      "learning_rate": 1.401198464962021e-06,
+      "logits/chosen": -1.3610389232635498,
+      "logits/rejected": -1.198563814163208,
+      "logps/chosen": -530.4500732421875,
+      "logps/rejected": -605.023681640625,
+      "loss": 0.4649,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.6202147006988525,
+      "rewards/margins": 1.1309864521026611,
+      "rewards/rejected": -3.7512009143829346,
+      "step": 2600
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "eval_logits/chosen": -1.2114638090133667,
+      "eval_logits/rejected": -1.079287052154541,
+      "eval_logps/chosen": -522.1907958984375,
+      "eval_logps/rejected": -632.8793334960938,
+      "eval_loss": 0.48811665177345276,
+      "eval_rewards/accuracies": 0.7450000047683716,
+      "eval_rewards/chosen": -2.5758883953094482,
+      "eval_rewards/margins": 1.307201623916626,
+      "eval_rewards/rejected": -3.883090019226074,
+      "eval_runtime": 1597.3459,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 2600
+    },
+    {
+      "epoch": 0.6830672598796127,
+      "grad_norm": 9.867508888244629,
+      "learning_rate": 1.3807248417879896e-06,
+      "logits/chosen": -1.401888132095337,
+      "logits/rejected": -1.276673436164856,
+      "logps/chosen": -532.1458740234375,
+      "logps/rejected": -652.4072265625,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.591766357421875,
+      "rewards/margins": 1.404159426689148,
+      "rewards/rejected": -3.9959254264831543,
+      "step": 2610
+    },
+    {
+      "epoch": 0.6856843758178487,
+      "grad_norm": 28.170438766479492,
+      "learning_rate": 1.3603446777675665e-06,
+      "logits/chosen": -1.209559440612793,
+      "logits/rejected": -1.0783087015151978,
+      "logps/chosen": -564.7611694335938,
+      "logps/rejected": -670.5452270507812,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.0164246559143066,
+      "rewards/margins": 1.2787444591522217,
+      "rewards/rejected": -4.295169353485107,
+      "step": 2620
+    },
+    {
+      "epoch": 0.6883014917560848,
+      "grad_norm": 9.69133472442627,
+      "learning_rate": 1.3400596746385817e-06,
+      "logits/chosen": -1.3486610651016235,
+      "logits/rejected": -1.1891324520111084,
+      "logps/chosen": -538.7862548828125,
+      "logps/rejected": -633.6087646484375,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.6963882446289062,
+      "rewards/margins": 1.2195419073104858,
+      "rewards/rejected": -3.9159302711486816,
+      "step": 2630
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 11.340239524841309,
+      "learning_rate": 1.3198715261929587e-06,
+      "logits/chosen": -1.339261531829834,
+      "logits/rejected": -1.186591386795044,
+      "logps/chosen": -508.19189453125,
+      "logps/rejected": -621.9693603515625,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.717308759689331,
+      "rewards/margins": 1.2740724086761475,
+      "rewards/rejected": -3.9913814067840576,
+      "step": 2640
+    },
+    {
+      "epoch": 0.6935357236325569,
+      "grad_norm": 8.626651763916016,
+      "learning_rate": 1.2997819181352823e-06,
+      "logits/chosen": -1.3360286951065063,
+      "logits/rejected": -1.1647727489471436,
+      "logps/chosen": -564.6326904296875,
+      "logps/rejected": -691.1988525390625,
+      "loss": 0.4326,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.678508996963501,
+      "rewards/margins": 1.511307954788208,
+      "rewards/rejected": -4.189816474914551,
+      "step": 2650
+    },
+    {
+      "epoch": 0.696152839570793,
+      "grad_norm": 21.902114868164062,
+      "learning_rate": 1.2797925279420454e-06,
+      "logits/chosen": -1.2936923503875732,
+      "logits/rejected": -1.1569340229034424,
+      "logps/chosen": -576.9588623046875,
+      "logps/rejected": -705.35791015625,
+      "loss": 0.4632,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.0516982078552246,
+      "rewards/margins": 1.4037137031555176,
+      "rewards/rejected": -4.4554123878479,
+      "step": 2660
+    },
+    {
+      "epoch": 0.6987699555090291,
+      "grad_norm": 12.999093055725098,
+      "learning_rate": 1.2599050247215764e-06,
+      "logits/chosen": -1.2631757259368896,
+      "logits/rejected": -1.1437580585479736,
+      "logps/chosen": -550.2919921875,
+      "logps/rejected": -659.2994384765625,
+      "loss": 0.4911,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.9243297576904297,
+      "rewards/margins": 1.3196732997894287,
+      "rewards/rejected": -4.244002819061279,
+      "step": 2670
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 15.106201171875,
+      "learning_rate": 1.2401210690746705e-06,
+      "logits/chosen": -1.2861721515655518,
+      "logits/rejected": -1.1344573497772217,
+      "logps/chosen": -562.5394287109375,
+      "logps/rejected": -647.7611083984375,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.9545741081237793,
+      "rewards/margins": 1.1845426559448242,
+      "rewards/rejected": -4.1391167640686035,
+      "step": 2680
+    },
+    {
+      "epoch": 0.7040041873855012,
+      "grad_norm": 9.087705612182617,
+      "learning_rate": 1.2204423129559306e-06,
+      "logits/chosen": -1.3246266841888428,
+      "logits/rejected": -1.2674678564071655,
+      "logps/chosen": -540.13037109375,
+      "logps/rejected": -657.595947265625,
+      "loss": 0.492,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.7469382286071777,
+      "rewards/margins": 1.2162708044052124,
+      "rewards/rejected": -3.963209629058838,
+      "step": 2690
+    },
+    {
+      "epoch": 0.7066213033237373,
+      "grad_norm": 18.654743194580078,
+      "learning_rate": 1.20087039953583e-06,
+      "logits/chosen": -1.3834110498428345,
+      "logits/rejected": -1.2643133401870728,
+      "logps/chosen": -513.6646728515625,
+      "logps/rejected": -604.9307250976562,
+      "loss": 0.556,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.4955801963806152,
+      "rewards/margins": 1.1764256954193115,
+      "rewards/rejected": -3.6720058917999268,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7066213033237373,
+      "eval_logits/chosen": -1.2295472621917725,
+      "eval_logits/rejected": -1.1003633737564087,
+      "eval_logps/chosen": -498.9264831542969,
+      "eval_logps/rejected": -595.6959228515625,
+      "eval_loss": 0.4841165840625763,
+      "eval_rewards/accuracies": 0.7459999918937683,
+      "eval_rewards/chosen": -2.343245029449463,
+      "eval_rewards/margins": 1.1680108308792114,
+      "eval_rewards/rejected": -3.5112557411193848,
+      "eval_runtime": 1597.0057,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7092384192619733,
+      "grad_norm": 19.703975677490234,
+      "learning_rate": 1.181406963063507e-06,
+      "logits/chosen": -1.3118432760238647,
+      "logits/rejected": -1.2396559715270996,
+      "logps/chosen": -480.10662841796875,
+      "logps/rejected": -590.1580200195312,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.1435210704803467,
+      "rewards/margins": 1.1070889234542847,
+      "rewards/rejected": -3.250609874725342,
+      "step": 2710
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 5.974539279937744,
+      "learning_rate": 1.1620536287303052e-06,
+      "logits/chosen": -1.4100855588912964,
+      "logits/rejected": -1.2877388000488281,
+      "logps/chosen": -500.7962951660156,
+      "logps/rejected": -562.1715087890625,
+      "loss": 0.5414,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.097153902053833,
+      "rewards/margins": 0.9155322909355164,
+      "rewards/rejected": -3.012686014175415,
+      "step": 2720
+    },
+    {
+      "epoch": 0.7144726511384454,
+      "grad_norm": 9.851229667663574,
+      "learning_rate": 1.1428120125340717e-06,
+      "logits/chosen": -1.3525193929672241,
+      "logits/rejected": -1.19950532913208,
+      "logps/chosen": -445.181884765625,
+      "logps/rejected": -556.6793212890625,
+      "loss": 0.3931,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.911560297012329,
+      "rewards/margins": 1.5294151306152344,
+      "rewards/rejected": -3.4409751892089844,
+      "step": 2730
+    },
+    {
+      "epoch": 0.7170897670766815,
+      "grad_norm": 9.149617195129395,
+      "learning_rate": 1.123683721144223e-06,
+      "logits/chosen": -1.3333719968795776,
+      "logits/rejected": -1.2258195877075195,
+      "logps/chosen": -494.1949157714844,
+      "logps/rejected": -600.24658203125,
+      "loss": 0.4166,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.106128692626953,
+      "rewards/margins": 1.385012149810791,
+      "rewards/rejected": -3.491140842437744,
+      "step": 2740
+    },
+    {
+      "epoch": 0.7197068830149176,
+      "grad_norm": 7.582621097564697,
+      "learning_rate": 1.1046703517675848e-06,
+      "logits/chosen": -1.3422666788101196,
+      "logits/rejected": -1.2579666376113892,
+      "logps/chosen": -470.9822692871094,
+      "logps/rejected": -587.7799072265625,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.159165620803833,
+      "rewards/margins": 1.106806993484497,
+      "rewards/rejected": -3.26597261428833,
+      "step": 2750
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 12.026503562927246,
+      "learning_rate": 1.085773492015028e-06,
+      "logits/chosen": -1.3507201671600342,
+      "logits/rejected": -1.1788911819458008,
+      "logps/chosen": -452.8814392089844,
+      "logps/rejected": -557.76416015625,
+      "loss": 0.4254,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.065864324569702,
+      "rewards/margins": 1.3924720287322998,
+      "rewards/rejected": -3.458336353302002,
+      "step": 2760
+    },
+    {
+      "epoch": 0.7249411148913897,
+      "grad_norm": 16.423751831054688,
+      "learning_rate": 1.0669947197689034e-06,
+      "logits/chosen": -1.3033778667449951,
+      "logits/rejected": -1.1522514820098877,
+      "logps/chosen": -498.0997619628906,
+      "logps/rejected": -590.888427734375,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.2554616928100586,
+      "rewards/margins": 1.2149760723114014,
+      "rewards/rejected": -3.470437526702881,
+      "step": 2770
+    },
+    {
+      "epoch": 0.7275582308296258,
+      "grad_norm": 8.218092918395996,
+      "learning_rate": 1.048335603051291e-06,
+      "logits/chosen": -1.2994598150253296,
+      "logits/rejected": -1.1608821153640747,
+      "logps/chosen": -527.0745849609375,
+      "logps/rejected": -638.7930908203125,
+      "loss": 0.4518,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.3942160606384277,
+      "rewards/margins": 1.3985862731933594,
+      "rewards/rejected": -3.792802333831787,
+      "step": 2780
+    },
+    {
+      "epoch": 0.7301753467678618,
+      "grad_norm": 12.542786598205566,
+      "learning_rate": 1.0297976998930665e-06,
+      "logits/chosen": -1.2890033721923828,
+      "logits/rejected": -1.1566731929779053,
+      "logps/chosen": -490.1463928222656,
+      "logps/rejected": -611.6651611328125,
+      "loss": 0.4472,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.296391010284424,
+      "rewards/margins": 1.492492437362671,
+      "rewards/rejected": -3.7888832092285156,
+      "step": 2790
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 10.247793197631836,
+      "learning_rate": 1.0113825582038078e-06,
+      "logits/chosen": -1.329484224319458,
+      "logits/rejected": -1.2037220001220703,
+      "logps/chosen": -503.18951416015625,
+      "logps/rejected": -605.0428466796875,
+      "loss": 0.4617,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.380659580230713,
+      "rewards/margins": 1.1856248378753662,
+      "rewards/rejected": -3.5662841796875,
+      "step": 2800
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "eval_logits/chosen": -1.195982813835144,
+      "eval_logits/rejected": -1.0626633167266846,
+      "eval_logps/chosen": -499.5495910644531,
+      "eval_logps/rejected": -606.4032592773438,
+      "eval_loss": 0.4832090735435486,
+      "eval_rewards/accuracies": 0.7459999918937683,
+      "eval_rewards/chosen": -2.3494763374328613,
+      "eval_rewards/margins": 1.2688524723052979,
+      "eval_rewards/rejected": -3.61832857131958,
+      "eval_runtime": 1596.6931,
+      "eval_samples_per_second": 1.253,
+      "eval_steps_per_second": 0.157,
+      "step": 2800
+    },
+    {
+      "epoch": 0.735409578644334,
+      "grad_norm": 20.173498153686523,
+      "learning_rate": 9.930917156425477e-07,
+      "logits/chosen": -1.2981245517730713,
+      "logits/rejected": -1.1898633241653442,
+      "logps/chosen": -514.4564819335938,
+      "logps/rejected": -633.3460693359375,
+      "loss": 0.5311,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.5339112281799316,
+      "rewards/margins": 1.2178757190704346,
+      "rewards/rejected": -3.751786708831787,
+      "step": 2810
+    },
+    {
+      "epoch": 0.73802669458257,
+      "grad_norm": 20.3377742767334,
+      "learning_rate": 9.749266994893756e-07,
+      "logits/chosen": -1.2338041067123413,
+      "logits/rejected": -1.084707498550415,
+      "logps/chosen": -491.3687438964844,
+      "logps/rejected": -575.6991577148438,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.5079922676086426,
+      "rewards/margins": 0.9769840240478516,
+      "rewards/rejected": -3.484976291656494,
+      "step": 2820
+    },
+    {
+      "epoch": 0.7406438105208061,
+      "grad_norm": 10.66334056854248,
+      "learning_rate": 9.56889026517913e-07,
+      "logits/chosen": -1.2951042652130127,
+      "logits/rejected": -1.1750590801239014,
+      "logps/chosen": -511.2039489746094,
+      "logps/rejected": -602.0439453125,
+      "loss": 0.4889,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.499505043029785,
+      "rewards/margins": 1.1682642698287964,
+      "rewards/rejected": -3.66776967048645,
+      "step": 2830
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 8.688248634338379,
+      "learning_rate": 9.389802028686617e-07,
+      "logits/chosen": -1.371790885925293,
+      "logits/rejected": -1.270684003829956,
+      "logps/chosen": -503.0506896972656,
+      "logps/rejected": -553.471923828125,
+      "loss": 0.5805,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.423825979232788,
+      "rewards/margins": 0.8341992497444153,
+      "rewards/rejected": -3.2580254077911377,
+      "step": 2840
+    },
+    {
+      "epoch": 0.7458780423972782,
+      "grad_norm": 13.514219284057617,
+      "learning_rate": 9.212017239232427e-07,
+      "logits/chosen": -1.2997102737426758,
+      "logits/rejected": -1.1494895219802856,
+      "logps/chosen": -497.5753479003906,
+      "logps/rejected": -604.2395629882812,
+      "loss": 0.4702,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.2467424869537354,
+      "rewards/margins": 1.2961435317993164,
+      "rewards/rejected": -3.542886257171631,
+      "step": 2850
+    },
+    {
+      "epoch": 0.7484951583355143,
+      "grad_norm": 10.475621223449707,
+      "learning_rate": 9.03555074179533e-07,
+      "logits/chosen": -1.2712374925613403,
+      "logits/rejected": -1.236365795135498,
+      "logps/chosen": -477.6539611816406,
+      "logps/rejected": -613.723876953125,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.210069179534912,
+      "rewards/margins": 1.3023548126220703,
+      "rewards/rejected": -3.5124244689941406,
+      "step": 2860
+    },
+    {
+      "epoch": 0.7511122742737504,
+      "grad_norm": 11.980389595031738,
+      "learning_rate": 8.860417271277067e-07,
+      "logits/chosen": -1.3676173686981201,
+      "logits/rejected": -1.303006887435913,
+      "logps/chosen": -500.90008544921875,
+      "logps/rejected": -591.3121948242188,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.3170719146728516,
+      "rewards/margins": 0.9840759038925171,
+      "rewards/rejected": -3.3011481761932373,
+      "step": 2870
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 8.44219970703125,
+      "learning_rate": 8.686631451272029e-07,
+      "logits/chosen": -1.323055624961853,
+      "logits/rejected": -1.1673837900161743,
+      "logps/chosen": -512.1810302734375,
+      "logps/rejected": -617.2755126953125,
+      "loss": 0.4958,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.588519811630249,
+      "rewards/margins": 1.3053323030471802,
+      "rewards/rejected": -3.8938522338867188,
+      "step": 2880
+    },
+    {
+      "epoch": 0.7563465061502225,
+      "grad_norm": 6.950828552246094,
+      "learning_rate": 8.514207792846168e-07,
+      "logits/chosen": -1.3387699127197266,
+      "logits/rejected": -1.2082509994506836,
+      "logps/chosen": -511.2001953125,
+      "logps/rejected": -603.291748046875,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.6037611961364746,
+      "rewards/margins": 1.2057433128356934,
+      "rewards/rejected": -3.809504747390747,
+      "step": 2890
+    },
+    {
+      "epoch": 0.7589636220884585,
+      "grad_norm": 7.15659236907959,
+      "learning_rate": 8.343160693325356e-07,
+      "logits/chosen": -1.2071359157562256,
+      "logits/rejected": -1.0866135358810425,
+      "logps/chosen": -522.898193359375,
+      "logps/rejected": -642.6585083007812,
+      "loss": 0.4916,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.6741576194763184,
+      "rewards/margins": 1.2413873672485352,
+      "rewards/rejected": -3.9155445098876953,
+      "step": 2900
+    },
+    {
+      "epoch": 0.7589636220884585,
+      "eval_logits/chosen": -1.1417629718780518,
+      "eval_logits/rejected": -1.0031887292861938,
+      "eval_logps/chosen": -531.7142333984375,
+      "eval_logps/rejected": -636.2195434570312,
+      "eval_loss": 0.4799574017524719,
+      "eval_rewards/accuracies": 0.7455000281333923,
+      "eval_rewards/chosen": -2.6711227893829346,
+      "eval_rewards/margins": 1.2453694343566895,
+      "eval_rewards/rejected": -3.916492223739624,
+      "eval_runtime": 1597.0819,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 2900
+    },
+    {
+      "epoch": 0.7615807380266946,
+      "grad_norm": 8.567606925964355,
+      "learning_rate": 8.173504435093174e-07,
+      "logits/chosen": -1.228169560432434,
+      "logits/rejected": -1.035468339920044,
+      "logps/chosen": -499.9391174316406,
+      "logps/rejected": -596.357666015625,
+      "loss": 0.483,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.584282398223877,
+      "rewards/margins": 1.2857377529144287,
+      "rewards/rejected": -3.8700199127197266,
+      "step": 2910
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 8.268507957458496,
+      "learning_rate": 8.00525318439836e-07,
+      "logits/chosen": -1.2574290037155151,
+      "logits/rejected": -1.1231721639633179,
+      "logps/chosen": -538.8193969726562,
+      "logps/rejected": -632.8656005859375,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.62135648727417,
+      "rewards/margins": 1.0157769918441772,
+      "rewards/rejected": -3.637133836746216,
+      "step": 2920
+    },
+    {
+      "epoch": 0.7668149699031667,
+      "grad_norm": 6.73305606842041,
+      "learning_rate": 7.838420990171927e-07,
+      "logits/chosen": -1.3410282135009766,
+      "logits/rejected": -1.1771111488342285,
+      "logps/chosen": -512.6897583007812,
+      "logps/rejected": -595.9611206054688,
+      "loss": 0.5007,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.4339559078216553,
+      "rewards/margins": 1.0879731178283691,
+      "rewards/rejected": -3.5219292640686035,
+      "step": 2930
+    },
+    {
+      "epoch": 0.7694320858414028,
+      "grad_norm": 8.820892333984375,
+      "learning_rate": 7.673021782854084e-07,
+      "logits/chosen": -1.195052981376648,
+      "logits/rejected": -1.0439643859863281,
+      "logps/chosen": -517.55810546875,
+      "logps/rejected": -611.39111328125,
+      "loss": 0.4715,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.5697178840637207,
+      "rewards/margins": 1.3570950031280518,
+      "rewards/rejected": -3.9268131256103516,
+      "step": 2940
+    },
+    {
+      "epoch": 0.7720492017796389,
+      "grad_norm": 10.797639846801758,
+      "learning_rate": 7.509069373231039e-07,
+      "logits/chosen": -1.2065623998641968,
+      "logits/rejected": -1.0761216878890991,
+      "logps/chosen": -521.6849365234375,
+      "logps/rejected": -586.7667236328125,
+      "loss": 0.5756,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.6908392906188965,
+      "rewards/margins": 0.898908257484436,
+      "rewards/rejected": -3.589747667312622,
+      "step": 2950
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 9.844982147216797,
+      "learning_rate": 7.346577451281822e-07,
+      "logits/chosen": -1.219063639640808,
+      "logits/rejected": -1.133622407913208,
+      "logps/chosen": -516.6390991210938,
+      "logps/rejected": -624.2025146484375,
+      "loss": 0.4668,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.544440269470215,
+      "rewards/margins": 1.317181944847107,
+      "rewards/rejected": -3.8616223335266113,
+      "step": 2960
+    },
+    {
+      "epoch": 0.777283433656111,
+      "grad_norm": 12.88412094116211,
+      "learning_rate": 7.185559585035138e-07,
+      "logits/chosen": -1.269226312637329,
+      "logits/rejected": -1.0988438129425049,
+      "logps/chosen": -547.8538208007812,
+      "logps/rejected": -646.4948120117188,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.6517109870910645,
+      "rewards/margins": 1.145399570465088,
+      "rewards/rejected": -3.7971103191375732,
+      "step": 2970
+    },
+    {
+      "epoch": 0.7799005495943471,
+      "grad_norm": 8.924067497253418,
+      "learning_rate": 7.026029219436504e-07,
+      "logits/chosen": -1.2982590198516846,
+      "logits/rejected": -1.1203540563583374,
+      "logps/chosen": -504.6836853027344,
+      "logps/rejected": -623.8721313476562,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.5236408710479736,
+      "rewards/margins": 1.3211889266967773,
+      "rewards/rejected": -3.844829559326172,
+      "step": 2980
+    },
+    {
+      "epoch": 0.7825176655325831,
+      "grad_norm": 6.5459513664245605,
+      "learning_rate": 6.867999675225523e-07,
+      "logits/chosen": -1.3317945003509521,
+      "logits/rejected": -1.1817572116851807,
+      "logps/chosen": -483.4960021972656,
+      "logps/rejected": -588.3883056640625,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.526824712753296,
+      "rewards/margins": 1.1909806728363037,
+      "rewards/rejected": -3.7178053855895996,
+      "step": 2990
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 8.477150917053223,
+      "learning_rate": 6.711484147823663e-07,
+      "logits/chosen": -1.2142189741134644,
+      "logits/rejected": -1.1248340606689453,
+      "logps/chosen": -488.151123046875,
+      "logps/rejected": -618.1202392578125,
+      "loss": 0.4708,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.5399701595306396,
+      "rewards/margins": 1.251422643661499,
+      "rewards/rejected": -3.7913928031921387,
+      "step": 3000
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "eval_logits/chosen": -1.1355363130569458,
+      "eval_logits/rejected": -0.9962058663368225,
+      "eval_logps/chosen": -526.2620849609375,
+      "eval_logps/rejected": -623.4008178710938,
+      "eval_loss": 0.4796713590621948,
+      "eval_rewards/accuracies": 0.7475000023841858,
+      "eval_rewards/chosen": -2.61660099029541,
+      "eval_rewards/margins": 1.1717036962509155,
+      "eval_rewards/rejected": -3.788304328918457,
+      "eval_runtime": 1596.8532,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 3000
+    },
+    {
+      "epoch": 0.7877518974090553,
+      "grad_norm": 9.99060344696045,
+      "learning_rate": 6.556495706232413e-07,
+      "logits/chosen": -1.2248764038085938,
+      "logits/rejected": -1.1362669467926025,
+      "logps/chosen": -536.1866455078125,
+      "logps/rejected": -627.3114013671875,
+      "loss": 0.532,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.6758596897125244,
+      "rewards/margins": 1.0958218574523926,
+      "rewards/rejected": -3.771681308746338,
+      "step": 3010
+    },
+    {
+      "epoch": 0.7903690133472913,
+      "grad_norm": 10.53675365447998,
+      "learning_rate": 6.403047291942057e-07,
+      "logits/chosen": -1.1557037830352783,
+      "logits/rejected": -0.9910370111465454,
+      "logps/chosen": -491.5957946777344,
+      "logps/rejected": -585.2208251953125,
+      "loss": 0.4845,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.6533892154693604,
+      "rewards/margins": 1.1808980703353882,
+      "rewards/rejected": -3.834287166595459,
+      "step": 3020
+    },
+    {
+      "epoch": 0.7929861292855274,
+      "grad_norm": 12.331343650817871,
+      "learning_rate": 6.251151717851023e-07,
+      "logits/chosen": -1.2338765859603882,
+      "logits/rejected": -1.136103630065918,
+      "logps/chosen": -477.06243896484375,
+      "logps/rejected": -573.8302001953125,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.4373462200164795,
+      "rewards/margins": 1.100510835647583,
+      "rewards/rejected": -3.5378570556640625,
+      "step": 3030
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 6.849348545074463,
+      "learning_rate": 6.100821667196041e-07,
+      "logits/chosen": -1.4195866584777832,
+      "logits/rejected": -1.1471283435821533,
+      "logps/chosen": -517.2559814453125,
+      "logps/rejected": -562.2097778320312,
+      "loss": 0.4849,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.4172825813293457,
+      "rewards/margins": 1.1046142578125,
+      "rewards/rejected": -3.5218968391418457,
+      "step": 3040
+    },
+    {
+      "epoch": 0.7982203611619995,
+      "grad_norm": 7.553493022918701,
+      "learning_rate": 5.952069692493062e-07,
+      "logits/chosen": -1.2353808879852295,
+      "logits/rejected": -1.117495059967041,
+      "logps/chosen": -466.36297607421875,
+      "logps/rejected": -601.0093994140625,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.381345272064209,
+      "rewards/margins": 1.3122522830963135,
+      "rewards/rejected": -3.6935970783233643,
+      "step": 3050
+    },
+    {
+      "epoch": 0.8008374771002356,
+      "grad_norm": 9.807490348815918,
+      "learning_rate": 5.80490821448918e-07,
+      "logits/chosen": -1.1880605220794678,
+      "logits/rejected": -1.173678994178772,
+      "logps/chosen": -516.3980102539062,
+      "logps/rejected": -701.3677978515625,
+      "loss": 0.4261,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.5393073558807373,
+      "rewards/margins": 1.4251797199249268,
+      "rewards/rejected": -3.9644875526428223,
+      "step": 3060
+    },
+    {
+      "epoch": 0.8034545930384716,
+      "grad_norm": 8.029556274414062,
+      "learning_rate": 5.659349521125459e-07,
+      "logits/chosen": -1.3551298379898071,
+      "logits/rejected": -1.2928274869918823,
+      "logps/chosen": -548.3309326171875,
+      "logps/rejected": -636.1687622070312,
+      "loss": 0.5164,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.670652389526367,
+      "rewards/margins": 1.083081603050232,
+      "rewards/rejected": -3.7537341117858887,
+      "step": 3070
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 10.774998664855957,
+      "learning_rate": 5.5154057665109e-07,
+      "logits/chosen": -1.3124372959136963,
+      "logits/rejected": -1.1519577503204346,
+      "logps/chosen": -552.2019653320312,
+      "logps/rejected": -666.1383666992188,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.9138247966766357,
+      "rewards/margins": 1.403515338897705,
+      "rewards/rejected": -4.31734037399292,
+      "step": 3080
+    },
+    {
+      "epoch": 0.8086888249149438,
+      "grad_norm": 6.534247398376465,
+      "learning_rate": 5.373088969907586e-07,
+      "logits/chosen": -1.3402431011199951,
+      "logits/rejected": -1.147871732711792,
+      "logps/chosen": -560.8900756835938,
+      "logps/rejected": -637.8880004882812,
+      "loss": 0.4401,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.8420605659484863,
+      "rewards/margins": 1.2714219093322754,
+      "rewards/rejected": -4.113482475280762,
+      "step": 3090
+    },
+    {
+      "epoch": 0.8113059408531798,
+      "grad_norm": 7.741410732269287,
+      "learning_rate": 5.23241101472709e-07,
+      "logits/chosen": -1.2486029863357544,
+      "logits/rejected": -1.1143968105316162,
+      "logps/chosen": -550.2741088867188,
+      "logps/rejected": -650.1898803710938,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.740609645843506,
+      "rewards/margins": 1.2194688320159912,
+      "rewards/rejected": -3.960078001022339,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8113059408531798,
+      "eval_logits/chosen": -1.1341181993484497,
+      "eval_logits/rejected": -0.9953464865684509,
+      "eval_logps/chosen": -546.843505859375,
+      "eval_logps/rejected": -656.7727661132812,
+      "eval_loss": 0.4807169735431671,
+      "eval_rewards/accuracies": 0.7475000023841858,
+      "eval_rewards/chosen": -2.822416067123413,
+      "eval_rewards/margins": 1.2996082305908203,
+      "eval_rewards/rejected": -4.122024059295654,
+      "eval_runtime": 1596.5373,
+      "eval_samples_per_second": 1.253,
+      "eval_steps_per_second": 0.157,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8139230567914159,
+      "grad_norm": 12.117105484008789,
+      "learning_rate": 5.09338364753818e-07,
+      "logits/chosen": -1.3339240550994873,
+      "logits/rejected": -1.1453027725219727,
+      "logps/chosen": -560.3875122070312,
+      "logps/rejected": -672.76513671875,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.750267267227173,
+      "rewards/margins": 1.282500982284546,
+      "rewards/rejected": -4.032768249511719,
+      "step": 3110
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 9.4435396194458,
+      "learning_rate": 4.956018477086005e-07,
+      "logits/chosen": -1.2970499992370605,
+      "logits/rejected": -1.1291049718856812,
+      "logps/chosen": -554.2510986328125,
+      "logps/rejected": -649.1993408203125,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.851862668991089,
+      "rewards/margins": 1.2229183912277222,
+      "rewards/rejected": -4.07478141784668,
+      "step": 3120
+    },
+    {
+      "epoch": 0.819157288667888,
+      "grad_norm": 14.918078422546387,
+      "learning_rate": 4.820326973322764e-07,
+      "logits/chosen": -1.207421064376831,
+      "logits/rejected": -1.115781307220459,
+      "logps/chosen": -536.1851806640625,
+      "logps/rejected": -655.1953735351562,
+      "loss": 0.5314,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.8535518646240234,
+      "rewards/margins": 1.2501590251922607,
+      "rewards/rejected": -4.103711128234863,
+      "step": 3130
+    },
+    {
+      "epoch": 0.821774404606124,
+      "grad_norm": 14.224996566772461,
+      "learning_rate": 4.686320466449981e-07,
+      "logits/chosen": -1.1797192096710205,
+      "logits/rejected": -0.9966877102851868,
+      "logps/chosen": -504.07745361328125,
+      "logps/rejected": -650.1423950195312,
+      "loss": 0.4351,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.6490511894226074,
+      "rewards/margins": 1.4823657274246216,
+      "rewards/rejected": -4.131417274475098,
+      "step": 3140
+    },
+    {
+      "epoch": 0.8243915205443602,
+      "grad_norm": 7.642600059509277,
+      "learning_rate": 4.554010145972418e-07,
+      "logits/chosen": -1.366541862487793,
+      "logits/rejected": -1.1694148778915405,
+      "logps/chosen": -543.9080200195312,
+      "logps/rejected": -658.1217041015625,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.8025176525115967,
+      "rewards/margins": 1.2542707920074463,
+      "rewards/rejected": -4.056788444519043,
+      "step": 3150
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 10.804971694946289,
+      "learning_rate": 4.4234070597637455e-07,
+      "logits/chosen": -1.1909892559051514,
+      "logits/rejected": -1.0947834253311157,
+      "logps/chosen": -546.0935668945312,
+      "logps/rejected": -647.5870361328125,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.70656156539917,
+      "rewards/margins": 1.1345676183700562,
+      "rewards/rejected": -3.8411293029785156,
+      "step": 3160
+    },
+    {
+      "epoch": 0.8296257524208323,
+      "grad_norm": 6.038959980010986,
+      "learning_rate": 4.2945221131440783e-07,
+      "logits/chosen": -1.1933271884918213,
+      "logits/rejected": -0.9797855615615845,
+      "logps/chosen": -524.9500732421875,
+      "logps/rejected": -634.7277221679688,
+      "loss": 0.4177,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.5928843021392822,
+      "rewards/margins": 1.4146864414215088,
+      "rewards/rejected": -4.007569789886475,
+      "step": 3170
+    },
+    {
+      "epoch": 0.8322428683590684,
+      "grad_norm": 8.643757820129395,
+      "learning_rate": 4.167366067969381e-07,
+      "logits/chosen": -1.3083521127700806,
+      "logits/rejected": -1.2275665998458862,
+      "logps/chosen": -480.93267822265625,
+      "logps/rejected": -616.6661376953125,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.5809030532836914,
+      "rewards/margins": 1.1199270486831665,
+      "rewards/rejected": -3.7008299827575684,
+      "step": 3180
+    },
+    {
+      "epoch": 0.8348599842973043,
+      "grad_norm": 5.735963821411133,
+      "learning_rate": 4.041949541732826e-07,
+      "logits/chosen": -1.306793451309204,
+      "logits/rejected": -1.2211034297943115,
+      "logps/chosen": -528.7879638671875,
+      "logps/rejected": -633.7890625,
+      "loss": 0.5016,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.649538040161133,
+      "rewards/margins": 1.1934170722961426,
+      "rewards/rejected": -3.8429553508758545,
+      "step": 3190
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 11.755797386169434,
+      "learning_rate": 3.9182830066782614e-07,
+      "logits/chosen": -1.2019577026367188,
+      "logits/rejected": -1.1839076280593872,
+      "logps/chosen": -523.3551025390625,
+      "logps/rejected": -663.1622314453125,
+      "loss": 0.4866,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.657108783721924,
+      "rewards/margins": 1.273402452468872,
+      "rewards/rejected": -3.930511474609375,
+      "step": 3200
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "eval_logits/chosen": -1.1640751361846924,
+      "eval_logits/rejected": -1.0276466608047485,
+      "eval_logps/chosen": -519.5614013671875,
+      "eval_logps/rejected": -623.51025390625,
+      "eval_loss": 0.4776689112186432,
+      "eval_rewards/accuracies": 0.7475000023841858,
+      "eval_rewards/chosen": -2.5495944023132324,
+      "eval_rewards/margins": 1.2398039102554321,
+      "eval_rewards/rejected": -3.789398431777954,
+      "eval_runtime": 1596.1554,
+      "eval_samples_per_second": 1.253,
+      "eval_steps_per_second": 0.157,
+      "step": 3200
+    },
+    {
+      "epoch": 0.8400942161737766,
+      "grad_norm": 6.5980448722839355,
+      "learning_rate": 3.796376788925771e-07,
+      "logits/chosen": -1.2055485248565674,
+      "logits/rejected": -1.1355664730072021,
+      "logps/chosen": -506.8805236816406,
+      "logps/rejected": -596.1102294921875,
+      "loss": 0.503,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.4550302028656006,
+      "rewards/margins": 1.1328961849212646,
+      "rewards/rejected": -3.5879263877868652,
+      "step": 3210
+    },
+    {
+      "epoch": 0.8427113321120125,
+      "grad_norm": 7.025763034820557,
+      "learning_rate": 3.676241067609465e-07,
+      "logits/chosen": -1.2890058755874634,
+      "logits/rejected": -1.1607505083084106,
+      "logps/chosen": -542.5242919921875,
+      "logps/rejected": -612.6026611328125,
+      "loss": 0.5185,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.488643169403076,
+      "rewards/margins": 1.1322394609451294,
+      "rewards/rejected": -3.620882749557495,
+      "step": 3220
+    },
+    {
+      "epoch": 0.8453284480502486,
+      "grad_norm": 11.517923355102539,
+      "learning_rate": 3.5578858740274976e-07,
+      "logits/chosen": -1.2237865924835205,
+      "logits/rejected": -1.122482419013977,
+      "logps/chosen": -517.73974609375,
+      "logps/rejected": -605.0042114257812,
+      "loss": 0.513,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.5605525970458984,
+      "rewards/margins": 0.999243438243866,
+      "rewards/rejected": -3.5597965717315674,
+      "step": 3230
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 10.88183307647705,
+      "learning_rate": 3.44132109080447e-07,
+      "logits/chosen": -1.4292676448822021,
+      "logits/rejected": -1.2410900592803955,
+      "logps/chosen": -495.89813232421875,
+      "logps/rejected": -589.0921630859375,
+      "loss": 0.4351,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.3068690299987793,
+      "rewards/margins": 1.3076789379119873,
+      "rewards/rejected": -3.6145482063293457,
+      "step": 3240
+    },
+    {
+      "epoch": 0.8505626799267207,
+      "grad_norm": 11.31247615814209,
+      "learning_rate": 3.3265564510662344e-07,
+      "logits/chosen": -1.3701411485671997,
+      "logits/rejected": -1.213030457496643,
+      "logps/chosen": -523.7352294921875,
+      "logps/rejected": -627.9140625,
+      "loss": 0.433,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.305351495742798,
+      "rewards/margins": 1.2927907705307007,
+      "rewards/rejected": -3.598142147064209,
+      "step": 3250
+    },
+    {
+      "epoch": 0.8531797958649568,
+      "grad_norm": 15.092782974243164,
+      "learning_rate": 3.213601537627195e-07,
+      "logits/chosen": -1.2755637168884277,
+      "logits/rejected": -1.1552997827529907,
+      "logps/chosen": -517.78369140625,
+      "logps/rejected": -615.5398559570312,
+      "loss": 0.5176,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.607300281524658,
+      "rewards/margins": 1.1482493877410889,
+      "rewards/rejected": -3.755549669265747,
+      "step": 3260
+    },
+    {
+      "epoch": 0.8557969118031928,
+      "grad_norm": 18.888704299926758,
+      "learning_rate": 3.1024657821901063e-07,
+      "logits/chosen": -1.3271687030792236,
+      "logits/rejected": -1.2444849014282227,
+      "logps/chosen": -477.8196716308594,
+      "logps/rejected": -587.2836303710938,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.301666498184204,
+      "rewards/margins": 1.2528338432312012,
+      "rewards/rejected": -3.554500102996826,
+      "step": 3270
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 10.203365325927734,
+      "learning_rate": 2.9931584645585654e-07,
+      "logits/chosen": -1.245596170425415,
+      "logits/rejected": -1.222507357597351,
+      "logps/chosen": -513.1766357421875,
+      "logps/rejected": -632.6079711914062,
+      "loss": 0.4915,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.386977434158325,
+      "rewards/margins": 1.1650269031524658,
+      "rewards/rejected": -3.55200457572937,
+      "step": 3280
+    },
+    {
+      "epoch": 0.861031143679665,
+      "grad_norm": 4.975100040435791,
+      "learning_rate": 2.885688711862136e-07,
+      "logits/chosen": -1.2813748121261597,
+      "logits/rejected": -1.2710316181182861,
+      "logps/chosen": -519.4554443359375,
+      "logps/rejected": -658.6768188476562,
+      "loss": 0.498,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.585131883621216,
+      "rewards/margins": 1.4114031791687012,
+      "rewards/rejected": -3.996534824371338,
+      "step": 3290
+    },
+    {
+      "epoch": 0.863648259617901,
+      "grad_norm": 6.192923069000244,
+      "learning_rate": 2.7800654977942486e-07,
+      "logits/chosen": -1.268638014793396,
+      "logits/rejected": -1.1421396732330322,
+      "logps/chosen": -508.80584716796875,
+      "logps/rejected": -629.664306640625,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.516002893447876,
+      "rewards/margins": 1.2464697360992432,
+      "rewards/rejected": -3.762472629547119,
+      "step": 3300
+    },
+    {
+      "epoch": 0.863648259617901,
+      "eval_logits/chosen": -1.1608073711395264,
+      "eval_logits/rejected": -1.0241122245788574,
+      "eval_logps/chosen": -520.3804321289062,
+      "eval_logps/rejected": -625.6535034179688,
+      "eval_loss": 0.47857987880706787,
+      "eval_rewards/accuracies": 0.7480000257492065,
+      "eval_rewards/chosen": -2.5577852725982666,
+      "eval_rewards/margins": 1.2530462741851807,
+      "eval_rewards/rejected": -3.8108315467834473,
+      "eval_runtime": 1596.9029,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 3300
+    },
+    {
+      "epoch": 0.8662653755561371,
+      "grad_norm": 14.262938499450684,
+      "learning_rate": 2.6762976418628797e-07,
+      "logits/chosen": -1.2763694524765015,
+      "logits/rejected": -1.12501859664917,
+      "logps/chosen": -479.6934509277344,
+      "logps/rejected": -549.134765625,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.4996352195739746,
+      "rewards/margins": 1.1307785511016846,
+      "rewards/rejected": -3.630413770675659,
+      "step": 3310
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 6.411558628082275,
+      "learning_rate": 2.5743938086541354e-07,
+      "logits/chosen": -1.277998447418213,
+      "logits/rejected": -1.1551088094711304,
+      "logps/chosen": -519.344482421875,
+      "logps/rejected": -614.8687744140625,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.5514492988586426,
+      "rewards/margins": 1.2041881084442139,
+      "rewards/rejected": -3.7556369304656982,
+      "step": 3320
+    },
+    {
+      "epoch": 0.8714996074326092,
+      "grad_norm": 8.243268966674805,
+      "learning_rate": 2.4743625071087574e-07,
+      "logits/chosen": -1.444610834121704,
+      "logits/rejected": -1.2527801990509033,
+      "logps/chosen": -515.1294555664062,
+      "logps/rejected": -632.4005737304688,
+      "loss": 0.4532,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.376555919647217,
+      "rewards/margins": 1.4679479598999023,
+      "rewards/rejected": -3.844503879547119,
+      "step": 3330
+    },
+    {
+      "epoch": 0.8741167233708453,
+      "grad_norm": 10.0430908203125,
+      "learning_rate": 2.3762120898116498e-07,
+      "logits/chosen": -1.2998971939086914,
+      "logits/rejected": -1.1857765913009644,
+      "logps/chosen": -535.6951293945312,
+      "logps/rejected": -634.7950439453125,
+      "loss": 0.495,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.6694915294647217,
+      "rewards/margins": 1.0715022087097168,
+      "rewards/rejected": -3.7409939765930176,
+      "step": 3340
+    },
+    {
+      "epoch": 0.8767338393090814,
+      "grad_norm": 9.759672164916992,
+      "learning_rate": 2.2799507522944048e-07,
+      "logits/chosen": -1.2516978979110718,
+      "logits/rejected": -1.143761396408081,
+      "logps/chosen": -506.92779541015625,
+      "logps/rejected": -630.4886474609375,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.4124372005462646,
+      "rewards/margins": 1.3180114030838013,
+      "rewards/rejected": -3.7304489612579346,
+      "step": 3350
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 10.614029884338379,
+      "learning_rate": 2.1855865323510056e-07,
+      "logits/chosen": -1.3051875829696655,
+      "logits/rejected": -1.0945546627044678,
+      "logps/chosen": -524.2318115234375,
+      "logps/rejected": -672.577880859375,
+      "loss": 0.4244,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.515920639038086,
+      "rewards/margins": 1.5423296689987183,
+      "rewards/rejected": -4.058249473571777,
+      "step": 3360
+    },
+    {
+      "epoch": 0.8819680711855535,
+      "grad_norm": 7.248552322387695,
+      "learning_rate": 2.0931273093666575e-07,
+      "logits/chosen": -1.2330162525177002,
+      "logits/rejected": -1.0763533115386963,
+      "logps/chosen": -503.97393798828125,
+      "logps/rejected": -626.6051025390625,
+      "loss": 0.4239,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.638221025466919,
+      "rewards/margins": 1.4292933940887451,
+      "rewards/rejected": -4.067514419555664,
+      "step": 3370
+    },
+    {
+      "epoch": 0.8845851871237895,
+      "grad_norm": 15.31811237335205,
+      "learning_rate": 2.002580803659873e-07,
+      "logits/chosen": -1.244128942489624,
+      "logits/rejected": -1.1012144088745117,
+      "logps/chosen": -522.0372924804688,
+      "logps/rejected": -629.9783325195312,
+      "loss": 0.4693,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.676211357116699,
+      "rewards/margins": 1.2609608173370361,
+      "rewards/rejected": -3.9371724128723145,
+      "step": 3380
+    },
+    {
+      "epoch": 0.8872023030620256,
+      "grad_norm": 5.824941635131836,
+      "learning_rate": 1.913954575837826e-07,
+      "logits/chosen": -1.3020130395889282,
+      "logits/rejected": -1.055781602859497,
+      "logps/chosen": -542.0621337890625,
+      "logps/rejected": -616.4013061523438,
+      "loss": 0.4735,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.7002296447753906,
+      "rewards/margins": 1.2402583360671997,
+      "rewards/rejected": -3.9404876232147217,
+      "step": 3390
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 9.155735969543457,
+      "learning_rate": 1.827256026165028e-07,
+      "logits/chosen": -1.3028671741485596,
+      "logits/rejected": -1.1115076541900635,
+      "logps/chosen": -565.5799560546875,
+      "logps/rejected": -653.9213256835938,
+      "loss": 0.4272,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.5059080123901367,
+      "rewards/margins": 1.455091118812561,
+      "rewards/rejected": -3.960999011993408,
+      "step": 3400
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "eval_logits/chosen": -1.1445426940917969,
+      "eval_logits/rejected": -1.0071464776992798,
+      "eval_logps/chosen": -536.8281860351562,
+      "eval_logps/rejected": -647.4435424804688,
+      "eval_loss": 0.47965455055236816,
+      "eval_rewards/accuracies": 0.7459999918937683,
+      "eval_rewards/chosen": -2.722262382507324,
+      "eval_rewards/margins": 1.306469440460205,
+      "eval_rewards/rejected": -4.0287322998046875,
+      "eval_runtime": 1597.1181,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.157,
+      "step": 3400
+    },
+    {
+      "epoch": 0.8924365349384977,
+      "grad_norm": 9.588942527770996,
+      "learning_rate": 1.7424923939454274e-07,
+      "logits/chosen": -1.2525078058242798,
+      "logits/rejected": -1.0694096088409424,
+      "logps/chosen": -553.6488037109375,
+      "logps/rejected": -648.3309326171875,
+      "loss": 0.4282,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.723491668701172,
+      "rewards/margins": 1.380472183227539,
+      "rewards/rejected": -4.103963851928711,
+      "step": 3410
+    },
+    {
+      "epoch": 0.8950536508767338,
+      "grad_norm": 19.193740844726562,
+      "learning_rate": 1.6596707569179304e-07,
+      "logits/chosen": -1.3545329570770264,
+      "logits/rejected": -1.1915156841278076,
+      "logps/chosen": -562.5875244140625,
+      "logps/rejected": -653.5098876953125,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.815075635910034,
+      "rewards/margins": 1.2605375051498413,
+      "rewards/rejected": -4.075612545013428,
+      "step": 3420
+    },
+    {
+      "epoch": 0.8976707668149699,
+      "grad_norm": 14.299762725830078,
+      "learning_rate": 1.578798030665385e-07,
+      "logits/chosen": -1.283125638961792,
+      "logits/rejected": -1.0879403352737427,
+      "logps/chosen": -551.2545166015625,
+      "logps/rejected": -682.09228515625,
+      "loss": 0.4447,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.7663490772247314,
+      "rewards/margins": 1.4793423414230347,
+      "rewards/rejected": -4.245691776275635,
+      "step": 3430
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 8.79478645324707,
+      "learning_rate": 1.499880968037165e-07,
+      "logits/chosen": -1.2685706615447998,
+      "logits/rejected": -1.1165311336517334,
+      "logps/chosen": -530.1332397460938,
+      "logps/rejected": -616.2289428710938,
+      "loss": 0.5132,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.7463672161102295,
+      "rewards/margins": 1.2237727642059326,
+      "rewards/rejected": -3.970139980316162,
+      "step": 3440
+    },
+    {
+      "epoch": 0.902904998691442,
+      "grad_norm": 14.623788833618164,
+      "learning_rate": 1.4229261585852805e-07,
+      "logits/chosen": -1.298165202140808,
+      "logits/rejected": -1.205263614654541,
+      "logps/chosen": -537.5665283203125,
+      "logps/rejected": -645.2786865234375,
+      "loss": 0.4557,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.692805528640747,
+      "rewards/margins": 1.3347504138946533,
+      "rewards/rejected": -4.0275559425354,
+      "step": 3450
+    },
+    {
+      "epoch": 0.9055221146296781,
+      "grad_norm": 11.80216121673584,
+      "learning_rate": 1.3479400280141886e-07,
+      "logits/chosen": -1.2136515378952026,
+      "logits/rejected": -1.1657497882843018,
+      "logps/chosen": -525.5947265625,
+      "logps/rejected": -662.1110229492188,
+      "loss": 0.4657,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.7938077449798584,
+      "rewards/margins": 1.385801076889038,
+      "rewards/rejected": -4.1796088218688965,
+      "step": 3460
+    },
+    {
+      "epoch": 0.9081392305679141,
+      "grad_norm": 9.422633171081543,
+      "learning_rate": 1.2749288376442044e-07,
+      "logits/chosen": -1.3240694999694824,
+      "logits/rejected": -1.106979489326477,
+      "logps/chosen": -565.2491455078125,
+      "logps/rejected": -643.6472778320312,
+      "loss": 0.4631,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.6884024143218994,
+      "rewards/margins": 1.3586628437042236,
+      "rewards/rejected": -4.047064781188965,
+      "step": 3470
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 9.699939727783203,
+      "learning_rate": 1.203898683888713e-07,
+      "logits/chosen": -1.306654691696167,
+      "logits/rejected": -1.1677879095077515,
+      "logps/chosen": -532.5252685546875,
+      "logps/rejected": -648.9597778320312,
+      "loss": 0.5443,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.886373519897461,
+      "rewards/margins": 1.1954319477081299,
+      "rewards/rejected": -4.081805229187012,
+      "step": 3480
+    },
+    {
+      "epoch": 0.9133734624443863,
+      "grad_norm": 11.356287002563477,
+      "learning_rate": 1.1348554977451132e-07,
+      "logits/chosen": -1.3395811319351196,
+      "logits/rejected": -1.1923797130584717,
+      "logps/chosen": -551.8304443359375,
+      "logps/rejected": -638.4985961914062,
+      "loss": 0.5142,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.699251651763916,
+      "rewards/margins": 1.195854902267456,
+      "rewards/rejected": -3.895106792449951,
+      "step": 3490
+    },
+    {
+      "epoch": 0.9159905783826223,
+      "grad_norm": 10.900007247924805,
+      "learning_rate": 1.0678050442995802e-07,
+      "logits/chosen": -1.306223750114441,
+      "logits/rejected": -1.088254690170288,
+      "logps/chosen": -554.8663330078125,
+      "logps/rejected": -634.9073486328125,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.7420237064361572,
+      "rewards/margins": 1.2511793375015259,
+      "rewards/rejected": -3.9932029247283936,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9159905783826223,
+      "eval_logits/chosen": -1.160068154335022,
+      "eval_logits/rejected": -1.023296594619751,
+      "eval_logps/chosen": -536.0448608398438,
+      "eval_logps/rejected": -647.7730102539062,
+      "eval_loss": 0.4797233045101166,
+      "eval_rewards/accuracies": 0.746999979019165,
+      "eval_rewards/chosen": -2.7144289016723633,
+      "eval_rewards/margins": 1.3175978660583496,
+      "eval_rewards/rejected": -4.032026767730713,
+      "eval_runtime": 1597.7222,
+      "eval_samples_per_second": 1.252,
+      "eval_steps_per_second": 0.156,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9186076943208584,
+      "grad_norm": 9.065542221069336,
+      "learning_rate": 1.0027529222456755e-07,
+      "logits/chosen": -1.2851347923278809,
+      "logits/rejected": -1.0965713262557983,
+      "logps/chosen": -514.8778076171875,
+      "logps/rejected": -634.3299560546875,
+      "loss": 0.4078,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.6071436405181885,
+      "rewards/margins": 1.371249794960022,
+      "rewards/rejected": -3.978393077850342,
+      "step": 3510
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 12.434161186218262,
+      "learning_rate": 9.397045634168766e-08,
+      "logits/chosen": -1.3080555200576782,
+      "logits/rejected": -1.222429633140564,
+      "logps/chosen": -531.6901245117188,
+      "logps/rejected": -675.6932373046875,
+      "loss": 0.4615,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.632573366165161,
+      "rewards/margins": 1.43732750415802,
+      "rewards/rejected": -4.069900989532471,
+      "step": 3520
+    },
+    {
+      "epoch": 0.9238419261973305,
+      "grad_norm": 16.84272003173828,
+      "learning_rate": 8.78665232332998e-08,
+      "logits/chosen": -1.243849515914917,
+      "logits/rejected": -1.1529252529144287,
+      "logps/chosen": -504.6124572753906,
+      "logps/rejected": -625.7620849609375,
+      "loss": 0.474,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.7395050525665283,
+      "rewards/margins": 1.2195281982421875,
+      "rewards/rejected": -3.959033489227295,
+      "step": 3530
+    },
+    {
+      "epoch": 0.9264590421355666,
+      "grad_norm": 8.225701332092285,
+      "learning_rate": 8.196400257606208e-08,
+      "logits/chosen": -1.348672866821289,
+      "logits/rejected": -1.1730903387069702,
+      "logps/chosen": -544.8364868164062,
+      "logps/rejected": -697.6771240234375,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.6183691024780273,
+      "rewards/margins": 1.5808098316192627,
+      "rewards/rejected": -4.199179172515869,
+      "step": 3540
+    },
+    {
+      "epoch": 0.9290761580738026,
+      "grad_norm": 11.679130554199219,
+      "learning_rate": 7.626338722875076e-08,
+      "logits/chosen": -1.2883799076080322,
+      "logits/rejected": -1.2092903852462769,
+      "logps/chosen": -517.6767578125,
+      "logps/rejected": -647.9732666015625,
+      "loss": 0.4827,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.6364307403564453,
+      "rewards/margins": 1.2941230535507202,
+      "rewards/rejected": -3.930554151535034,
+      "step": 3550
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 5.509209632873535,
+      "learning_rate": 7.076515319110688e-08,
+      "logits/chosen": -1.283998727798462,
+      "logits/rejected": -1.1587374210357666,
+      "logps/chosen": -517.294677734375,
+      "logps/rejected": -615.6148071289062,
+      "loss": 0.5114,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.6106066703796387,
+      "rewards/margins": 1.3990845680236816,
+      "rewards/rejected": -4.00969123840332,
+      "step": 3560
+    },
+    {
+      "epoch": 0.9343103899502748,
+      "grad_norm": 7.372361660003662,
+      "learning_rate": 6.54697595640899e-08,
+      "logits/chosen": -1.3087977170944214,
+      "logits/rejected": -1.178120493888855,
+      "logps/chosen": -560.3231201171875,
+      "logps/rejected": -660.3911743164062,
+      "loss": 0.4774,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.6826512813568115,
+      "rewards/margins": 1.2511831521987915,
+      "rewards/rejected": -3.9338345527648926,
+      "step": 3570
+    },
+    {
+      "epoch": 0.9369275058885108,
+      "grad_norm": 9.634334564208984,
+      "learning_rate": 6.037764851154426e-08,
+      "logits/chosen": -1.3032505512237549,
+      "logits/rejected": -1.231890320777893,
+      "logps/chosen": -522.193603515625,
+      "logps/rejected": -652.8558349609375,
+      "loss": 0.4929,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.5825421810150146,
+      "rewards/margins": 1.2584049701690674,
+      "rewards/rejected": -3.840946912765503,
+      "step": 3580
+    },
+    {
+      "epoch": 0.9395446218267469,
+      "grad_norm": 7.305212497711182,
+      "learning_rate": 5.548924522327748e-08,
+      "logits/chosen": -1.2807940244674683,
+      "logits/rejected": -1.1494871377944946,
+      "logps/chosen": -517.2708740234375,
+      "logps/rejected": -632.7682495117188,
+      "loss": 0.4594,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.5563392639160156,
+      "rewards/margins": 1.3009912967681885,
+      "rewards/rejected": -3.857330799102783,
+      "step": 3590
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 13.96353530883789,
+      "learning_rate": 5.0804957879556915e-08,
+      "logits/chosen": -1.198561429977417,
+      "logits/rejected": -1.0997329950332642,
+      "logps/chosen": -484.185546875,
+      "logps/rejected": -620.9610595703125,
+      "loss": 0.4441,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.587231397628784,
+      "rewards/margins": 1.3218923807144165,
+      "rewards/rejected": -3.909123659133911,
+      "step": 3600
+    },
+    {
+      "epoch": 0.942161737764983,
+      "eval_logits/chosen": -1.164100170135498,
+      "eval_logits/rejected": -1.0277760028839111,
+      "eval_logps/chosen": -529.1943969726562,
+      "eval_logps/rejected": -639.7042846679688,
+      "eval_loss": 0.4790266156196594,
+      "eval_rewards/accuracies": 0.746999979019165,
+      "eval_rewards/chosen": -2.6459240913391113,
+      "eval_rewards/margins": 1.3054152727127075,
+      "eval_rewards/rejected": -3.9513394832611084,
+      "eval_runtime": 1598.6642,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 3600
+    },
+    {
+      "epoch": 0.944778853703219,
+      "grad_norm": 9.906332969665527,
+      "learning_rate": 4.632517761702815e-08,
+      "logits/chosen": -1.2350178956985474,
+      "logits/rejected": -1.0819157361984253,
+      "logps/chosen": -498.882568359375,
+      "logps/rejected": -640.2724609375,
+      "loss": 0.4371,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.635540008544922,
+      "rewards/margins": 1.536821722984314,
+      "rewards/rejected": -4.172361850738525,
+      "step": 3610
+    },
+    {
+      "epoch": 0.9473959696414551,
+      "grad_norm": 12.568668365478516,
+      "learning_rate": 4.205027849605359e-08,
+      "logits/chosen": -1.2538177967071533,
+      "logits/rejected": -1.1405234336853027,
+      "logps/chosen": -525.0173950195312,
+      "logps/rejected": -613.74560546875,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.771757125854492,
+      "rewards/margins": 1.1880303621292114,
+      "rewards/rejected": -3.959787368774414,
+      "step": 3620
+    },
+    {
+      "epoch": 0.9500130855796912,
+      "grad_norm": 8.876080513000488,
+      "learning_rate": 3.798061746947995e-08,
+      "logits/chosen": -1.373834252357483,
+      "logits/rejected": -1.228161334991455,
+      "logps/chosen": -527.4785766601562,
+      "logps/rejected": -619.109375,
+      "loss": 0.4818,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.6347904205322266,
+      "rewards/margins": 1.3071154356002808,
+      "rewards/rejected": -3.941905975341797,
+      "step": 3630
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 9.864373207092285,
+      "learning_rate": 3.411653435283158e-08,
+      "logits/chosen": -1.29331636428833,
+      "logits/rejected": -1.077043056488037,
+      "logps/chosen": -534.7817993164062,
+      "logps/rejected": -597.4425048828125,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.6166446208953857,
+      "rewards/margins": 1.1832177639007568,
+      "rewards/rejected": -3.7998623847961426,
+      "step": 3640
+    },
+    {
+      "epoch": 0.9552473174561633,
+      "grad_norm": 12.852518081665039,
+      "learning_rate": 3.04583517959367e-08,
+      "logits/chosen": -1.339290976524353,
+      "logits/rejected": -1.1780240535736084,
+      "logps/chosen": -494.443603515625,
+      "logps/rejected": -590.2618408203125,
+      "loss": 0.4506,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.4245123863220215,
+      "rewards/margins": 1.2823518514633179,
+      "rewards/rejected": -3.70686411857605,
+      "step": 3650
+    },
+    {
+      "epoch": 0.9578644333943994,
+      "grad_norm": 9.657252311706543,
+      "learning_rate": 2.7006375255985984e-08,
+      "logits/chosen": -1.281280755996704,
+      "logits/rejected": -1.2413251399993896,
+      "logps/chosen": -539.2777099609375,
+      "logps/rejected": -645.9277954101562,
+      "loss": 0.5566,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.7454452514648438,
+      "rewards/margins": 1.0612398386001587,
+      "rewards/rejected": -3.8066844940185547,
+      "step": 3660
+    },
+    {
+      "epoch": 0.9604815493326354,
+      "grad_norm": 12.267367362976074,
+      "learning_rate": 2.3760892972027328e-08,
+      "logits/chosen": -1.3983352184295654,
+      "logits/rejected": -1.225462794303894,
+      "logps/chosen": -544.6994018554688,
+      "logps/rejected": -638.8689575195312,
+      "loss": 0.5249,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.7232279777526855,
+      "rewards/margins": 1.2808904647827148,
+      "rewards/rejected": -4.0041184425354,
+      "step": 3670
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 13.94206714630127,
+      "learning_rate": 2.072217594089765e-08,
+      "logits/chosen": -1.2447559833526611,
+      "logits/rejected": -1.2226978540420532,
+      "logps/chosen": -527.1612548828125,
+      "logps/rejected": -661.6229248046875,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.6566879749298096,
+      "rewards/margins": 1.467563271522522,
+      "rewards/rejected": -4.124251365661621,
+      "step": 3680
+    },
+    {
+      "epoch": 0.9657157812091076,
+      "grad_norm": 8.289405822753906,
+      "learning_rate": 1.789047789459375e-08,
+      "logits/chosen": -1.3621820211410522,
+      "logits/rejected": -1.1523287296295166,
+      "logps/chosen": -576.5960693359375,
+      "logps/rejected": -663.8076782226562,
+      "loss": 0.5052,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.597318172454834,
+      "rewards/margins": 1.3620169162750244,
+      "rewards/rejected": -3.9593348503112793,
+      "step": 3690
+    },
+    {
+      "epoch": 0.9683328971473436,
+      "grad_norm": 7.745994567871094,
+      "learning_rate": 1.5266035279088708e-08,
+      "logits/chosen": -1.1971657276153564,
+      "logits/rejected": -1.0677430629730225,
+      "logps/chosen": -573.20361328125,
+      "logps/rejected": -680.00244140625,
+      "loss": 0.4823,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.7669663429260254,
+      "rewards/margins": 1.330165147781372,
+      "rewards/rejected": -4.097131729125977,
+      "step": 3700
+    },
+    {
+      "epoch": 0.9683328971473436,
+      "eval_logits/chosen": -1.1687482595443726,
+      "eval_logits/rejected": -1.0329276323318481,
+      "eval_logps/chosen": -527.3952026367188,
+      "eval_logps/rejected": -637.1880493164062,
+      "eval_loss": 0.47885680198669434,
+      "eval_rewards/accuracies": 0.7480000257492065,
+      "eval_rewards/chosen": -2.627932548522949,
+      "eval_rewards/margins": 1.298244595527649,
+      "eval_rewards/rejected": -3.9261767864227295,
+      "eval_runtime": 1598.696,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 3700
+    },
+    {
+      "epoch": 0.9709500130855797,
+      "grad_norm": 18.376056671142578,
+      "learning_rate": 1.2849067234584623e-08,
+      "logits/chosen": -1.182948112487793,
+      "logits/rejected": -1.10740065574646,
+      "logps/chosen": -496.62152099609375,
+      "logps/rejected": -624.2833862304688,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.603543281555176,
+      "rewards/margins": 1.3529905080795288,
+      "rewards/rejected": -3.956533908843994,
+      "step": 3710
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 9.365938186645508,
+      "learning_rate": 1.0639775577218625e-08,
+      "logits/chosen": -1.1770192384719849,
+      "logits/rejected": -1.0035854578018188,
+      "logps/chosen": -516.5242309570312,
+      "logps/rejected": -607.8884887695312,
+      "loss": 0.5282,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.6737780570983887,
+      "rewards/margins": 1.2736929655075073,
+      "rewards/rejected": -3.9474711418151855,
+      "step": 3720
+    },
+    {
+      "epoch": 0.9761842449620518,
+      "grad_norm": 11.24964427947998,
+      "learning_rate": 8.638344782207486e-09,
+      "logits/chosen": -1.1972987651824951,
+      "logits/rejected": -1.0941554307937622,
+      "logps/chosen": -499.6304626464844,
+      "logps/rejected": -600.4740600585938,
+      "loss": 0.4853,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.5446014404296875,
+      "rewards/margins": 1.2359497547149658,
+      "rewards/rejected": -3.7805511951446533,
+      "step": 3730
+    },
+    {
+      "epoch": 0.9788013609002879,
+      "grad_norm": 10.543977737426758,
+      "learning_rate": 6.84494196844715e-09,
+      "logits/chosen": -1.2653145790100098,
+      "logits/rejected": -1.1376771926879883,
+      "logps/chosen": -532.8594970703125,
+      "logps/rejected": -669.3706665039062,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.608578681945801,
+      "rewards/margins": 1.4806853532791138,
+      "rewards/rejected": -4.089264392852783,
+      "step": 3740
+    },
+    {
+      "epoch": 0.9814184768385239,
+      "grad_norm": 8.994680404663086,
+      "learning_rate": 5.259716884556121e-09,
+      "logits/chosen": -1.3201556205749512,
+      "logits/rejected": -1.1730735301971436,
+      "logps/chosen": -524.9544677734375,
+      "logps/rejected": -640.7349853515625,
+      "loss": 0.451,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.6050658226013184,
+      "rewards/margins": 1.3061447143554688,
+      "rewards/rejected": -3.911210536956787,
+      "step": 3750
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 9.462129592895508,
+      "learning_rate": 3.882801896372967e-09,
+      "logits/chosen": -1.3206380605697632,
+      "logits/rejected": -1.223382830619812,
+      "logps/chosen": -523.351318359375,
+      "logps/rejected": -627.8754272460938,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.58086895942688,
+      "rewards/margins": 1.3636963367462158,
+      "rewards/rejected": -3.9445652961730957,
+      "step": 3760
+    },
+    {
+      "epoch": 0.9866527087149961,
+      "grad_norm": 11.428803443908691,
+      "learning_rate": 2.7143119759026614e-09,
+      "logits/chosen": -1.3305742740631104,
+      "logits/rejected": -1.1541904211044312,
+      "logps/chosen": -536.9759521484375,
+      "logps/rejected": -636.7061157226562,
+      "loss": 0.4246,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.528337001800537,
+      "rewards/margins": 1.2519800662994385,
+      "rewards/rejected": -3.7803173065185547,
+      "step": 3770
+    },
+    {
+      "epoch": 0.9892698246532321,
+      "grad_norm": 10.81966781616211,
+      "learning_rate": 1.754344691717591e-09,
+      "logits/chosen": -1.2256147861480713,
+      "logits/rejected": -1.1796106100082397,
+      "logps/chosen": -514.3230590820312,
+      "logps/rejected": -642.4959716796875,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.645564556121826,
+      "rewards/margins": 1.0614073276519775,
+      "rewards/rejected": -3.7069716453552246,
+      "step": 3780
+    },
+    {
+      "epoch": 0.9918869405914682,
+      "grad_norm": 16.201265335083008,
+      "learning_rate": 1.0029802008096335e-09,
+      "logits/chosen": -1.2533369064331055,
+      "logits/rejected": -1.0869061946868896,
+      "logps/chosen": -542.9913330078125,
+      "logps/rejected": -650.4954833984375,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.687774181365967,
+      "rewards/margins": 1.3033173084259033,
+      "rewards/rejected": -3.99109148979187,
+      "step": 3790
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 7.363870143890381,
+      "learning_rate": 4.602812418974534e-10,
+      "logits/chosen": -1.3605427742004395,
+      "logits/rejected": -1.2153841257095337,
+      "logps/chosen": -546.0870971679688,
+      "logps/rejected": -652.19921875,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.6433169841766357,
+      "rewards/margins": 1.3066623210906982,
+      "rewards/rejected": -3.949979305267334,
+      "step": 3800
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "eval_logits/chosen": -1.1657898426055908,
+      "eval_logits/rejected": -1.0296279191970825,
+      "eval_logps/chosen": -526.756103515625,
+      "eval_logps/rejected": -636.4028930664062,
+      "eval_loss": 0.4788345396518707,
+      "eval_rewards/accuracies": 0.7475000023841858,
+      "eval_rewards/chosen": -2.62154221534729,
+      "eval_rewards/margins": 1.2967824935913086,
+      "eval_rewards/rejected": -3.9183249473571777,
+      "eval_runtime": 1598.3049,
+      "eval_samples_per_second": 1.251,
+      "eval_steps_per_second": 0.156,
+      "step": 3800
+    },
+    {
+      "epoch": 0.9971211724679403,
+      "grad_norm": 12.966708183288574,
+      "learning_rate": 1.2629313018819312e-10,
+      "logits/chosen": -1.2771844863891602,
+      "logits/rejected": -1.147637128829956,
+      "logps/chosen": -510.83001708984375,
+      "logps/rejected": -608.63037109375,
+      "loss": 0.503,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.5848309993743896,
+      "rewards/margins": 1.134263277053833,
+      "rewards/rejected": -3.7190945148468018,
+      "step": 3810
+    },
+    {
+      "epoch": 0.9997382884061764,
+      "grad_norm": 16.982664108276367,
+      "learning_rate": 1.0437535929996855e-12,
+      "logits/chosen": -1.262603998184204,
+      "logits/rejected": -1.0868072509765625,
+      "logps/chosen": -551.4014892578125,
+      "logps/rejected": -662.4216918945312,
+      "loss": 0.4568,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.6468563079833984,
+      "rewards/margins": 1.5578194856643677,
+      "rewards/rejected": -4.204675674438477,
+      "step": 3820
+    },
+    {
+      "epoch": 1.0,
+      "step": 3821,
+      "total_flos": 0.0,
+      "train_loss": 0.517807064771465,
+      "train_runtime": 164396.369,
+      "train_samples_per_second": 0.372,
+      "train_steps_per_second": 0.023
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3821,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}