{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.996886351842242,
  "eval_steps": 240,
  "global_step": 962,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0020757654385054488,
      "grad_norm": 7.452356109448157,
      "learning_rate": 5.154639175257731e-09,
      "logits/chosen": -2.730942726135254,
      "logits/rejected": -2.654609203338623,
      "logps/chosen": -350.489990234375,
      "logps/rejected": -325.546875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02075765438505449,
      "grad_norm": 7.866099026608714,
      "learning_rate": 5.154639175257731e-08,
      "logits/chosen": -2.7327775955200195,
      "logits/rejected": -2.734964609146118,
      "logps/chosen": -366.4884033203125,
      "logps/rejected": -412.2764892578125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4826388955116272,
      "rewards/chosen": 6.40038269921206e-05,
      "rewards/margins": 0.00040091515984386206,
      "rewards/rejected": -0.00033691132557578385,
      "step": 10
    },
    {
      "epoch": 0.04151530877010898,
      "grad_norm": 7.559166611883559,
      "learning_rate": 1.0309278350515462e-07,
      "logits/chosen": -2.717351198196411,
      "logits/rejected": -2.69411563873291,
      "logps/chosen": -378.9507141113281,
      "logps/rejected": -404.0965270996094,
      "loss": 0.6921,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.00041852169670164585,
      "rewards/margins": 0.0022167633287608624,
      "rewards/rejected": -0.0017982417484745383,
      "step": 20
    },
    {
      "epoch": 0.062272963155163466,
      "grad_norm": 7.615638482493831,
      "learning_rate": 1.5463917525773197e-07,
      "logits/chosen": -2.719243288040161,
      "logits/rejected": -2.7044143676757812,
      "logps/chosen": -365.67889404296875,
      "logps/rejected": -388.8330993652344,
      "loss": 0.6864,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.004637600388377905,
      "rewards/margins": 0.013907420448958874,
      "rewards/rejected": -0.009269820526242256,
      "step": 30
    },
    {
      "epoch": 0.08303061754021795,
      "grad_norm": 7.7707957616247505,
      "learning_rate": 2.0618556701030925e-07,
      "logits/chosen": -2.7188639640808105,
      "logits/rejected": -2.6840219497680664,
      "logps/chosen": -350.5911865234375,
      "logps/rejected": -365.0216979980469,
      "loss": 0.6693,
      "rewards/accuracies": 0.871874988079071,
      "rewards/chosen": 0.027326706796884537,
      "rewards/margins": 0.049114055931568146,
      "rewards/rejected": -0.02178734540939331,
      "step": 40
    },
    {
      "epoch": 0.10378827192527244,
      "grad_norm": 8.90458099154434,
      "learning_rate": 2.5773195876288655e-07,
      "logits/chosen": -2.7294702529907227,
      "logits/rejected": -2.7243564128875732,
      "logps/chosen": -358.9928894042969,
      "logps/rejected": -397.3983459472656,
      "loss": 0.63,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.07257182896137238,
      "rewards/margins": 0.14314065873622894,
      "rewards/rejected": -0.07056883722543716,
      "step": 50
    },
    {
      "epoch": 0.12454592631032693,
      "grad_norm": 9.98989696476554,
      "learning_rate": 3.0927835051546394e-07,
      "logits/chosen": -2.6951613426208496,
      "logits/rejected": -2.678609848022461,
      "logps/chosen": -348.3409729003906,
      "logps/rejected": -426.9418029785156,
      "loss": 0.5482,
      "rewards/accuracies": 0.878125011920929,
      "rewards/chosen": -0.011659199371933937,
      "rewards/margins": 0.36892449855804443,
      "rewards/rejected": -0.3805837035179138,
      "step": 60
    },
    {
      "epoch": 0.14530358069538143,
      "grad_norm": 14.516753369132225,
      "learning_rate": 3.608247422680412e-07,
      "logits/chosen": -2.7372078895568848,
      "logits/rejected": -2.6939454078674316,
      "logps/chosen": -451.07293701171875,
      "logps/rejected": -535.2464599609375,
      "loss": 0.4282,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.5447245836257935,
      "rewards/margins": 0.7660267353057861,
      "rewards/rejected": -1.3107513189315796,
      "step": 70
    },
    {
      "epoch": 0.1660612350804359,
      "grad_norm": 15.352248807358325,
      "learning_rate": 4.123711340206185e-07,
      "logits/chosen": -2.7028908729553223,
      "logits/rejected": -2.6827831268310547,
      "logps/chosen": -547.4126586914062,
      "logps/rejected": -697.9911499023438,
      "loss": 0.3596,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.5639097690582275,
      "rewards/margins": 1.3926727771759033,
      "rewards/rejected": -2.9565823078155518,
      "step": 80
    },
    {
      "epoch": 0.1868188894654904,
      "grad_norm": 17.157617837316515,
      "learning_rate": 4.639175257731959e-07,
      "logits/chosen": -2.664567708969116,
      "logits/rejected": -2.664062023162842,
      "logps/chosen": -561.7244262695312,
      "logps/rejected": -796.8342895507812,
      "loss": 0.3035,
      "rewards/accuracies": 0.878125011920929,
      "rewards/chosen": -1.9646422863006592,
      "rewards/margins": 2.149301052093506,
      "rewards/rejected": -4.113943576812744,
      "step": 90
    },
    {
      "epoch": 0.2075765438505449,
      "grad_norm": 15.724313068127296,
      "learning_rate": 4.999851606199919e-07,
      "logits/chosen": -2.607901096343994,
      "logits/rejected": -2.587402582168579,
      "logps/chosen": -576.0076904296875,
      "logps/rejected": -909.0657348632812,
      "loss": 0.263,
      "rewards/accuracies": 0.903124988079071,
      "rewards/chosen": -2.0050268173217773,
      "rewards/margins": 3.0768344402313232,
      "rewards/rejected": -5.0818610191345215,
      "step": 100
    },
    {
      "epoch": 0.2283341982355994,
      "grad_norm": 22.849604961849657,
      "learning_rate": 4.997213984244138e-07,
      "logits/chosen": -2.399500608444214,
      "logits/rejected": -2.2552199363708496,
      "logps/chosen": -620.6517333984375,
      "logps/rejected": -1053.1802978515625,
      "loss": 0.2333,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.491640329360962,
      "rewards/margins": 4.089051246643066,
      "rewards/rejected": -6.580691337585449,
      "step": 110
    },
    {
      "epoch": 0.24909185262065386,
      "grad_norm": 15.418275685296619,
      "learning_rate": 4.991282726678214e-07,
      "logits/chosen": -2.1379497051239014,
      "logits/rejected": -1.6612409353256226,
      "logps/chosen": -636.1170654296875,
      "logps/rejected": -1038.722412109375,
      "loss": 0.2114,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.4115161895751953,
      "rewards/margins": 3.842604875564575,
      "rewards/rejected": -6.254120826721191,
      "step": 120
    },
    {
      "epoch": 0.26984950700570837,
      "grad_norm": 22.122903405439125,
      "learning_rate": 4.982065656380468e-07,
      "logits/chosen": -1.8730089664459229,
      "logits/rejected": -1.1086432933807373,
      "logps/chosen": -624.3699951171875,
      "logps/rejected": -1083.5306396484375,
      "loss": 0.1978,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.4477767944335938,
      "rewards/margins": 4.317242622375488,
      "rewards/rejected": -6.76501989364624,
      "step": 130
    },
    {
      "epoch": 0.29060716139076287,
      "grad_norm": 13.724914658926766,
      "learning_rate": 4.969574929966689e-07,
      "logits/chosen": -1.063720941543579,
      "logits/rejected": 0.11310062557458878,
      "logps/chosen": -654.7889404296875,
      "logps/rejected": -1111.530517578125,
      "loss": 0.2171,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.8432438373565674,
      "rewards/margins": 4.339225769042969,
      "rewards/rejected": -7.182469844818115,
      "step": 140
    },
    {
      "epoch": 0.3113648157758173,
      "grad_norm": 20.111379920464472,
      "learning_rate": 4.953827021756488e-07,
      "logits/chosen": -1.147434949874878,
      "logits/rejected": 0.20881839096546173,
      "logps/chosen": -609.7511596679688,
      "logps/rejected": -1106.64990234375,
      "loss": 0.1719,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -2.4953980445861816,
      "rewards/margins": 4.772416591644287,
      "rewards/rejected": -7.267814636230469,
      "step": 150
    },
    {
      "epoch": 0.3321224701608718,
      "grad_norm": 20.712950851230556,
      "learning_rate": 4.93484270204492e-07,
      "logits/chosen": -0.3996439576148987,
      "logits/rejected": 0.9938270449638367,
      "logps/chosen": -663.2901000976562,
      "logps/rejected": -1266.5684814453125,
      "loss": 0.1789,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -2.7938601970672607,
      "rewards/margins": 5.7776288986206055,
      "rewards/rejected": -8.571489334106445,
      "step": 160
    },
    {
      "epoch": 0.3528801245459263,
      "grad_norm": 13.955815229008401,
      "learning_rate": 4.91264700970804e-07,
      "logits/chosen": -0.3857128620147705,
      "logits/rejected": 1.0256187915802002,
      "logps/chosen": -606.496337890625,
      "logps/rejected": -1150.380126953125,
      "loss": 0.1822,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -2.4765748977661133,
      "rewards/margins": 5.2469162940979,
      "rewards/rejected": -7.723490238189697,
      "step": 170
    },
    {
      "epoch": 0.3736377789309808,
      "grad_norm": 19.91766661692759,
      "learning_rate": 4.88726921917853e-07,
      "logits/chosen": -0.7232545614242554,
      "logits/rejected": 0.9214665293693542,
      "logps/chosen": -608.4518432617188,
      "logps/rejected": -1221.78955078125,
      "loss": 0.1832,
      "rewards/accuracies": 0.909375011920929,
      "rewards/chosen": -2.418121337890625,
      "rewards/margins": 5.937131881713867,
      "rewards/rejected": -8.355253219604492,
      "step": 180
    },
    {
      "epoch": 0.39439543331603527,
      "grad_norm": 16.024239064339824,
      "learning_rate": 4.858742801834942e-07,
      "logits/chosen": -0.32018381357192993,
      "logits/rejected": 1.0246493816375732,
      "logps/chosen": -667.6340942382812,
      "logps/rejected": -1306.54833984375,
      "loss": 0.1733,
      "rewards/accuracies": 0.934374988079071,
      "rewards/chosen": -2.8495688438415527,
      "rewards/margins": 6.024672985076904,
      "rewards/rejected": -8.874241828918457,
      "step": 190
    },
    {
      "epoch": 0.4151530877010898,
      "grad_norm": 12.349261100548174,
      "learning_rate": 4.827105381855496e-07,
      "logits/chosen": 0.25890278816223145,
      "logits/rejected": 1.507102370262146,
      "logps/chosen": -618.0540161132812,
      "logps/rejected": -1187.555419921875,
      "loss": 0.1662,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.504819393157959,
      "rewards/margins": 5.406851291656494,
      "rewards/rejected": -7.911670684814453,
      "step": 200
    },
    {
      "epoch": 0.4359107420861443,
      "grad_norm": 13.66810823452425,
      "learning_rate": 4.79239868659464e-07,
      "logits/chosen": -0.016678806394338608,
      "logits/rejected": 1.5381004810333252,
      "logps/chosen": -641.5460815429688,
      "logps/rejected": -1252.699951171875,
      "loss": 0.1584,
      "rewards/accuracies": 0.934374988079071,
      "rewards/chosen": -2.724513530731201,
      "rewards/margins": 5.9185709953308105,
      "rewards/rejected": -8.643084526062012,
      "step": 210
    },
    {
      "epoch": 0.4566683964711988,
      "grad_norm": 22.723399659159355,
      "learning_rate": 4.7546684915478443e-07,
      "logits/chosen": -0.5056034326553345,
      "logits/rejected": 1.3583004474639893,
      "logps/chosen": -627.7324829101562,
      "logps/rejected": -1324.6092529296875,
      "loss": 0.1696,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -2.6251590251922607,
      "rewards/margins": 6.788819789886475,
      "rewards/rejected": -9.413978576660156,
      "step": 220
    },
    {
      "epoch": 0.4774260508562532,
      "grad_norm": 25.512034717506292,
      "learning_rate": 4.7139645599771953e-07,
      "logits/chosen": -0.05699120834469795,
      "logits/rejected": 1.986132025718689,
      "logps/chosen": -660.3882446289062,
      "logps/rejected": -1403.30322265625,
      "loss": 0.1656,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.9933114051818848,
      "rewards/margins": 7.2729973793029785,
      "rewards/rejected": -10.266307830810547,
      "step": 230
    },
    {
      "epoch": 0.49818370524130773,
      "grad_norm": 19.374014963474004,
      "learning_rate": 4.6703405772774325e-07,
      "logits/chosen": -0.2160125970840454,
      "logits/rejected": 2.073151111602783,
      "logps/chosen": -610.6514892578125,
      "logps/rejected": -1350.219482421875,
      "loss": 0.1586,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -2.5249383449554443,
      "rewards/margins": 7.157525539398193,
      "rewards/rejected": -9.682464599609375,
      "step": 240
    },
    {
      "epoch": 0.49818370524130773,
      "eval_logits/chosen": -0.19751745462417603,
      "eval_logits/rejected": 1.597546935081482,
      "eval_logps/chosen": -692.0928344726562,
      "eval_logps/rejected": -1332.0889892578125,
      "eval_loss": 0.1689271330833435,
      "eval_rewards/accuracies": 0.9252451062202454,
      "eval_rewards/chosen": -2.6868460178375244,
      "eval_rewards/margins": 6.174641132354736,
      "eval_rewards/rejected": -8.861486434936523,
      "eval_runtime": 297.8773,
      "eval_samples_per_second": 21.791,
      "eval_steps_per_second": 0.342,
      "step": 240
    },
    {
      "epoch": 0.5189413596263622,
      "grad_norm": 12.036861238820864,
      "learning_rate": 4.6238540801689896e-07,
      "logits/chosen": -0.4214223325252533,
      "logits/rejected": 1.8403441905975342,
      "logps/chosen": -610.5186767578125,
      "logps/rejected": -1277.1844482421875,
      "loss": 0.1604,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.307227373123169,
      "rewards/margins": 6.4557318687438965,
      "rewards/rejected": -8.762959480285645,
      "step": 250
    },
    {
      "epoch": 0.5396990140114167,
      "grad_norm": 20.090786603415435,
      "learning_rate": 4.5745663808114316e-07,
      "logits/chosen": -0.743881106376648,
      "logits/rejected": 1.5020240545272827,
      "logps/chosen": -657.1211547851562,
      "logps/rejected": -1360.107666015625,
      "loss": 0.1638,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.839501142501831,
      "rewards/margins": 6.807010650634766,
      "rewards/rejected": -9.646512031555176,
      "step": 260
    },
    {
      "epoch": 0.5604566683964712,
      "grad_norm": 14.890524731187542,
      "learning_rate": 4.5225424859373684e-07,
      "logits/chosen": -0.38103678822517395,
      "logits/rejected": 1.8405994176864624,
      "logps/chosen": -600.785888671875,
      "logps/rejected": -1403.572998046875,
      "loss": 0.1493,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -2.4715404510498047,
      "rewards/margins": 7.713797569274902,
      "rewards/rejected": -10.185338020324707,
      "step": 270
    },
    {
      "epoch": 0.5812143227815257,
      "grad_norm": 15.737222130316104,
      "learning_rate": 4.467851011113515e-07,
      "logits/chosen": 0.3215886056423187,
      "logits/rejected": 2.3104662895202637,
      "logps/chosen": -629.4906005859375,
      "logps/rejected": -1383.751953125,
      "loss": 0.1571,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.5350632667541504,
      "rewards/margins": 7.314475059509277,
      "rewards/rejected": -9.849536895751953,
      "step": 280
    },
    {
      "epoch": 0.6019719771665801,
      "grad_norm": 23.328559288837457,
      "learning_rate": 4.410564090241966e-07,
      "logits/chosen": 0.4208109378814697,
      "logits/rejected": 2.2348339557647705,
      "logps/chosen": -675.2015991210938,
      "logps/rejected": -1364.4970703125,
      "loss": 0.1635,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -3.0044052600860596,
      "rewards/margins": 6.818602085113525,
      "rewards/rejected": -9.823007583618164,
      "step": 290
    },
    {
      "epoch": 0.6227296315516346,
      "grad_norm": 14.02162805247057,
      "learning_rate": 4.35075728042106e-07,
      "logits/chosen": -0.0718420147895813,
      "logits/rejected": 1.6788543462753296,
      "logps/chosen": -612.244873046875,
      "logps/rejected": -1253.4449462890625,
      "loss": 0.1528,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.3619751930236816,
      "rewards/margins": 6.16156530380249,
      "rewards/rejected": -8.523540496826172,
      "step": 300
    },
    {
      "epoch": 0.6434872859366891,
      "grad_norm": 13.851968293163813,
      "learning_rate": 4.2885094622913016e-07,
      "logits/chosen": -0.06367097049951553,
      "logits/rejected": 1.8669401407241821,
      "logps/chosen": -630.0944213867188,
      "logps/rejected": -1326.2972412109375,
      "loss": 0.1541,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.6027626991271973,
      "rewards/margins": 6.887024879455566,
      "rewards/rejected": -9.489786148071289,
      "step": 310
    },
    {
      "epoch": 0.6642449403217436,
      "grad_norm": 22.056173099963235,
      "learning_rate": 4.223902735997788e-07,
      "logits/chosen": -0.3920370638370514,
      "logits/rejected": 1.7357165813446045,
      "logps/chosen": -582.3004150390625,
      "logps/rejected": -1314.4351806640625,
      "loss": 0.1538,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.171776294708252,
      "rewards/margins": 7.103475093841553,
      "rewards/rejected": -9.275251388549805,
      "step": 320
    },
    {
      "epoch": 0.6850025947067981,
      "grad_norm": 10.33520679821666,
      "learning_rate": 4.157022312906352e-07,
      "logits/chosen": -1.2140147686004639,
      "logits/rejected": 1.2578856945037842,
      "logps/chosen": -645.0560302734375,
      "logps/rejected": -1458.661376953125,
      "loss": 0.1492,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.7255163192749023,
      "rewards/margins": 7.840676784515381,
      "rewards/rejected": -10.566194534301758,
      "step": 330
    },
    {
      "epoch": 0.7057602490918526,
      "grad_norm": 12.698623571384912,
      "learning_rate": 4.0879564032162425e-07,
      "logits/chosen": -1.3428138494491577,
      "logits/rejected": 1.8070862293243408,
      "logps/chosen": -719.0316162109375,
      "logps/rejected": -1663.4603271484375,
      "loss": 0.143,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.4557037353515625,
      "rewards/margins": 9.146575927734375,
      "rewards/rejected": -12.602280616760254,
      "step": 340
    },
    {
      "epoch": 0.7265179034769071,
      "grad_norm": 16.68939070839139,
      "learning_rate": 4.016796099617569e-07,
      "logits/chosen": -1.3726019859313965,
      "logits/rejected": 1.8845252990722656,
      "logps/chosen": -672.85693359375,
      "logps/rejected": -1494.5582275390625,
      "loss": 0.1538,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.1178722381591797,
      "rewards/margins": 7.914468288421631,
      "rewards/rejected": -11.032341003417969,
      "step": 350
    },
    {
      "epoch": 0.7472755578619616,
      "grad_norm": 14.783186894273188,
      "learning_rate": 3.9436352571469577e-07,
      "logits/chosen": -0.5545839071273804,
      "logits/rejected": 3.833822727203369,
      "logps/chosen": -830.2039184570312,
      "logps/rejected": -1744.631103515625,
      "loss": 0.1444,
      "rewards/accuracies": 0.934374988079071,
      "rewards/chosen": -4.657293796539307,
      "rewards/margins": 9.05556583404541,
      "rewards/rejected": -13.712858200073242,
      "step": 360
    },
    {
      "epoch": 0.768033212247016,
      "grad_norm": 15.755226269776646,
      "learning_rate": 3.868570369399893e-07,
      "logits/chosen": -0.6274186372756958,
      "logits/rejected": 3.4528141021728516,
      "logps/chosen": -799.2054443359375,
      "logps/rejected": -1723.1851806640625,
      "loss": 0.1458,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -4.323949337005615,
      "rewards/margins": 9.110595703125,
      "rewards/rejected": -13.434545516967773,
      "step": 370
    },
    {
      "epoch": 0.7887908666320705,
      "grad_norm": 11.452118871998977,
      "learning_rate": 3.791700441262987e-07,
      "logits/chosen": 0.8511954545974731,
      "logits/rejected": 4.691292762756348,
      "logps/chosen": -939.5437622070312,
      "logps/rejected": -2012.4154052734375,
      "loss": 0.1417,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.871208667755127,
      "rewards/margins": 10.141061782836914,
      "rewards/rejected": -16.012271881103516,
      "step": 380
    },
    {
      "epoch": 0.809548521017125,
      "grad_norm": 13.459005174890807,
      "learning_rate": 3.7131268583340515e-07,
      "logits/chosen": 0.3982798457145691,
      "logits/rejected": 4.274272441864014,
      "logps/chosen": -845.7362060546875,
      "logps/rejected": -1642.904541015625,
      "loss": 0.1523,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.7007269859313965,
      "rewards/margins": 7.76672887802124,
      "rewards/rejected": -12.467455863952637,
      "step": 390
    },
    {
      "epoch": 0.8303061754021795,
      "grad_norm": 13.869817366585716,
      "learning_rate": 3.632953253202198e-07,
      "logits/chosen": 1.5343601703643799,
      "logits/rejected": 4.5011491775512695,
      "logps/chosen": -881.2744140625,
      "logps/rejected": -1823.2939453125,
      "loss": 0.1449,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.160914421081543,
      "rewards/margins": 9.128256797790527,
      "rewards/rejected": -14.289172172546387,
      "step": 400
    },
    {
      "epoch": 0.851063829787234,
      "grad_norm": 14.502067608546383,
      "learning_rate": 3.551285368764321e-07,
      "logits/chosen": 1.5195695161819458,
      "logits/rejected": 4.168100357055664,
      "logps/chosen": -846.6906127929688,
      "logps/rejected": -1635.856689453125,
      "loss": 0.1445,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.812326431274414,
      "rewards/margins": 7.744412422180176,
      "rewards/rejected": -12.556737899780273,
      "step": 410
    },
    {
      "epoch": 0.8718214841722886,
      "grad_norm": 30.730295160965017,
      "learning_rate": 3.468230918758242e-07,
      "logits/chosen": -0.11532606929540634,
      "logits/rejected": 3.6724467277526855,
      "logps/chosen": -856.9993286132812,
      "logps/rejected": -1789.368408203125,
      "loss": 0.1521,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -4.82938289642334,
      "rewards/margins": 9.0027494430542,
      "rewards/rejected": -13.832133293151855,
      "step": 420
    },
    {
      "epoch": 0.892579138557343,
      "grad_norm": 15.361401417475852,
      "learning_rate": 3.383899445696477e-07,
      "logits/chosen": -0.19606542587280273,
      "logits/rejected": 3.2190608978271484,
      "logps/chosen": -914.5608520507812,
      "logps/rejected": -1852.5074462890625,
      "loss": 0.1317,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -5.41568660736084,
      "rewards/margins": 9.153519630432129,
      "rewards/rejected": -14.569204330444336,
      "step": 430
    },
    {
      "epoch": 0.9133367929423976,
      "grad_norm": 15.988866292668206,
      "learning_rate": 3.2984021763879756e-07,
      "logits/chosen": 0.24461348354816437,
      "logits/rejected": 3.7112534046173096,
      "logps/chosen": -1049.1207275390625,
      "logps/rejected": -2157.442138671875,
      "loss": 0.1472,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.739071846008301,
      "rewards/margins": 10.936814308166504,
      "rewards/rejected": -17.675886154174805,
      "step": 440
    },
    {
      "epoch": 0.934094447327452,
      "grad_norm": 16.98533210114085,
      "learning_rate": 3.211851875238408e-07,
      "logits/chosen": 0.16562503576278687,
      "logits/rejected": 4.111753940582275,
      "logps/chosen": -1058.0418701171875,
      "logps/rejected": -2127.619873046875,
      "loss": 0.1444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.909078121185303,
      "rewards/margins": 10.602731704711914,
      "rewards/rejected": -17.511810302734375,
      "step": 450
    },
    {
      "epoch": 0.9548521017125065,
      "grad_norm": 14.967379090940785,
      "learning_rate": 3.124362695522476e-07,
      "logits/chosen": 1.7813737392425537,
      "logits/rejected": 4.8395795822143555,
      "logps/chosen": -1155.03125,
      "logps/rejected": -2117.0537109375,
      "loss": 0.1396,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.852016448974609,
      "rewards/margins": 9.481895446777344,
      "rewards/rejected": -17.333911895751953,
      "step": 460
    },
    {
      "epoch": 0.975609756097561,
      "grad_norm": 14.307026095079875,
      "learning_rate": 3.036050028824415e-07,
      "logits/chosen": 1.5212550163269043,
      "logits/rejected": 4.2941412925720215,
      "logps/chosen": -1053.684814453125,
      "logps/rejected": -1923.693359375,
      "loss": 0.125,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -6.825259208679199,
      "rewards/margins": 8.460533142089844,
      "rewards/rejected": -15.285791397094727,
      "step": 470
    },
    {
      "epoch": 0.9963674104826155,
      "grad_norm": 11.916307943626432,
      "learning_rate": 2.9470303528452547e-07,
      "logits/chosen": 1.5992648601531982,
      "logits/rejected": 4.383261680603027,
      "logps/chosen": -1080.980224609375,
      "logps/rejected": -2100.404541015625,
      "loss": 0.1408,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -7.290544033050537,
      "rewards/margins": 9.89597225189209,
      "rewards/rejected": -17.1865177154541,
      "step": 480
    },
    {
      "epoch": 0.9963674104826155,
      "eval_logits/chosen": 0.5014842748641968,
      "eval_logits/rejected": 3.342353582382202,
      "eval_logps/chosen": -1083.2030029296875,
      "eval_logps/rejected": -2038.5933837890625,
      "eval_loss": 0.14824749529361725,
      "eval_rewards/accuracies": 0.9325980544090271,
      "eval_rewards/chosen": -6.597947597503662,
      "eval_rewards/margins": 9.328583717346191,
      "eval_rewards/rejected": -15.926531791687012,
      "eval_runtime": 296.0381,
      "eval_samples_per_second": 21.926,
      "eval_steps_per_second": 0.345,
      "step": 480
    },
    {
      "epoch": 1.01712506486767,
      "grad_norm": 14.456304387791999,
      "learning_rate": 2.8574210777775755e-07,
      "logits/chosen": 0.9456971287727356,
      "logits/rejected": 4.39624547958374,
      "logps/chosen": -1059.8154296875,
      "logps/rejected": -2144.65576171875,
      "loss": 0.0951,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -6.945745944976807,
      "rewards/margins": 10.675562858581543,
      "rewards/rejected": -17.621309280395508,
      "step": 490
    },
    {
      "epoch": 1.0378827192527245,
      "grad_norm": 17.681966171705884,
      "learning_rate": 2.767340391450384e-07,
      "logits/chosen": 1.6467583179473877,
      "logits/rejected": 5.070549011230469,
      "logps/chosen": -1184.4014892578125,
      "logps/rejected": -2638.23291015625,
      "loss": 0.0919,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -8.20818042755127,
      "rewards/margins": 14.303576469421387,
      "rewards/rejected": -22.511754989624023,
      "step": 500
    },
    {
      "epoch": 1.058640373637779,
      "grad_norm": 13.802679882016555,
      "learning_rate": 2.6769071034483407e-07,
      "logits/chosen": 1.1824378967285156,
      "logits/rejected": 4.564583778381348,
      "logps/chosen": -947.0963745117188,
      "logps/rejected": -2079.13916015625,
      "loss": 0.0879,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -5.809744358062744,
      "rewards/margins": 11.030488967895508,
      "rewards/rejected": -16.840232849121094,
      "step": 510
    },
    {
      "epoch": 1.0793980280228335,
      "grad_norm": 22.098742924019817,
      "learning_rate": 2.5862404884109365e-07,
      "logits/chosen": 1.4323368072509766,
      "logits/rejected": 4.83809757232666,
      "logps/chosen": -1047.9368896484375,
      "logps/rejected": -2401.321533203125,
      "loss": 0.0826,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -6.900972843170166,
      "rewards/margins": 13.123873710632324,
      "rewards/rejected": -20.02484703063965,
      "step": 520
    },
    {
      "epoch": 1.100155682407888,
      "grad_norm": 14.888695199480608,
      "learning_rate": 2.495460128718305e-07,
      "logits/chosen": 0.8169673681259155,
      "logits/rejected": 4.492055416107178,
      "logps/chosen": -1088.3392333984375,
      "logps/rejected": -2334.029052734375,
      "loss": 0.0775,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -7.015921592712402,
      "rewards/margins": 12.27374267578125,
      "rewards/rejected": -19.289663314819336,
      "step": 530
    },
    {
      "epoch": 1.1209133367929425,
      "grad_norm": 13.711286256989611,
      "learning_rate": 2.404685756771143e-07,
      "logits/chosen": 0.3249141275882721,
      "logits/rejected": 4.28688383102417,
      "logps/chosen": -1050.6568603515625,
      "logps/rejected": -2440.43212890625,
      "loss": 0.0831,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.925228118896484,
      "rewards/margins": 13.674964904785156,
      "rewards/rejected": -20.60019302368164,
      "step": 540
    },
    {
      "epoch": 1.141670991177997,
      "grad_norm": 12.310015587533016,
      "learning_rate": 2.314037097072764e-07,
      "logits/chosen": 0.325296014547348,
      "logits/rejected": 4.037835121154785,
      "logps/chosen": -1070.4940185546875,
      "logps/rejected": -2429.653076171875,
      "loss": 0.0774,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.072981357574463,
      "rewards/margins": 13.413686752319336,
      "rewards/rejected": -20.48666763305664,
      "step": 550
    },
    {
      "epoch": 1.1624286455630513,
      "grad_norm": 18.024933264685323,
      "learning_rate": 2.2236337083215723e-07,
      "logits/chosen": 0.973385214805603,
      "logits/rejected": 4.8588714599609375,
      "logps/chosen": -1165.290283203125,
      "logps/rejected": -2533.037841796875,
      "loss": 0.0806,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -7.950223445892334,
      "rewards/margins": 13.607198715209961,
      "rewards/rejected": -21.557422637939453,
      "step": 560
    },
    {
      "epoch": 1.183186299948106,
      "grad_norm": 39.199036754410095,
      "learning_rate": 2.13359482572222e-07,
      "logits/chosen": 0.9793283343315125,
      "logits/rejected": 4.486080169677734,
      "logps/chosen": -1073.2510986328125,
      "logps/rejected": -2294.072509765625,
      "loss": 0.0855,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -7.108448028564453,
      "rewards/margins": 11.980100631713867,
      "rewards/rejected": -19.088550567626953,
      "step": 570
    },
    {
      "epoch": 1.2039439543331603,
      "grad_norm": 13.491261159834455,
      "learning_rate": 2.044039203723423e-07,
      "logits/chosen": 0.23347489535808563,
      "logits/rejected": 3.7263119220733643,
      "logps/chosen": -997.2879028320312,
      "logps/rejected": -2139.56298828125,
      "loss": 0.0886,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.343294143676758,
      "rewards/margins": 11.091756820678711,
      "rewards/rejected": -17.435049057006836,
      "step": 580
    },
    {
      "epoch": 1.2247016087182148,
      "grad_norm": 22.750928538479595,
      "learning_rate": 1.955084959389864e-07,
      "logits/chosen": 0.4886883795261383,
      "logits/rejected": 4.1630377769470215,
      "logps/chosen": -1106.54052734375,
      "logps/rejected": -2491.51416015625,
      "loss": 0.0749,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -7.559281826019287,
      "rewards/margins": 13.47680377960205,
      "rewards/rejected": -21.036083221435547,
      "step": 590
    },
    {
      "epoch": 1.2454592631032693,
      "grad_norm": 20.593372405978258,
      "learning_rate": 1.866849416614753e-07,
      "logits/chosen": 0.24794098734855652,
      "logits/rejected": 4.261725425720215,
      "logps/chosen": -1190.895751953125,
      "logps/rejected": -2684.16943359375,
      "loss": 0.0762,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -8.191116333007812,
      "rewards/margins": 14.663787841796875,
      "rewards/rejected": -22.854902267456055,
      "step": 600
    },
    {
      "epoch": 1.2662169174883238,
      "grad_norm": 23.08708475257412,
      "learning_rate": 1.7794489513785227e-07,
      "logits/chosen": -0.35908642411231995,
      "logits/rejected": 3.719221830368042,
      "logps/chosen": -1067.587158203125,
      "logps/rejected": -2360.556396484375,
      "loss": 0.0824,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.811041355133057,
      "rewards/margins": 12.751296997070312,
      "rewards/rejected": -19.562339782714844,
      "step": 610
    },
    {
      "epoch": 1.2869745718733783,
      "grad_norm": 11.158342147450194,
      "learning_rate": 1.692998838257744e-07,
      "logits/chosen": -0.18085989356040955,
      "logits/rejected": 3.5346386432647705,
      "logps/chosen": -1038.219970703125,
      "logps/rejected": -2255.41064453125,
      "loss": 0.0769,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.670246124267578,
      "rewards/margins": 11.845807075500488,
      "rewards/rejected": -18.51605224609375,
      "step": 620
    },
    {
      "epoch": 1.3077322262584328,
      "grad_norm": 14.33248304938314,
      "learning_rate": 1.6076130983867191e-07,
      "logits/chosen": -0.1000831350684166,
      "logits/rejected": 4.0322370529174805,
      "logps/chosen": -1037.5277099609375,
      "logps/rejected": -2512.465087890625,
      "loss": 0.0774,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.664424896240234,
      "rewards/margins": 14.430699348449707,
      "rewards/rejected": -21.095125198364258,
      "step": 630
    },
    {
      "epoch": 1.3284898806434873,
      "grad_norm": 18.5366504084505,
      "learning_rate": 1.5234043490722587e-07,
      "logits/chosen": -0.20596277713775635,
      "logits/rejected": 3.613173246383667,
      "logps/chosen": -1079.144287109375,
      "logps/rejected": -2312.60791015625,
      "loss": 0.0821,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -7.167913913726807,
      "rewards/margins": 12.191060066223145,
      "rewards/rejected": -19.35897445678711,
      "step": 640
    },
    {
      "epoch": 1.3492475350285418,
      "grad_norm": 18.228054814234156,
      "learning_rate": 1.44048365526001e-07,
      "logits/chosen": 0.0638003945350647,
      "logits/rejected": 3.9387619495391846,
      "logps/chosen": -1077.5164794921875,
      "logps/rejected": -2451.49462890625,
      "loss": 0.0867,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.139754295349121,
      "rewards/margins": 13.3494873046875,
      "rewards/rejected": -20.489240646362305,
      "step": 650
    },
    {
      "epoch": 1.3700051894135963,
      "grad_norm": 16.31608491567421,
      "learning_rate": 1.3589603830482243e-07,
      "logits/chosen": -0.08248420059680939,
      "logits/rejected": 4.198277950286865,
      "logps/chosen": -1142.9498291015625,
      "logps/rejected": -2611.378662109375,
      "loss": 0.0703,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.643826484680176,
      "rewards/margins": 14.516764640808105,
      "rewards/rejected": -22.160587310791016,
      "step": 660
    },
    {
      "epoch": 1.3907628437986508,
      "grad_norm": 21.850286306048936,
      "learning_rate": 1.2789420554421821e-07,
      "logits/chosen": -0.33774855732917786,
      "logits/rejected": 3.5582516193389893,
      "logps/chosen": -1178.3541259765625,
      "logps/rejected": -2523.93701171875,
      "loss": 0.0845,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -8.179405212402344,
      "rewards/margins": 13.182377815246582,
      "rewards/rejected": -21.36178207397461,
      "step": 670
    },
    {
      "epoch": 1.4115204981837053,
      "grad_norm": 15.996775283562163,
      "learning_rate": 1.200534210539509e-07,
      "logits/chosen": -0.7213211059570312,
      "logits/rejected": 3.0649337768554688,
      "logps/chosen": -1151.722412109375,
      "logps/rejected": -2552.525634765625,
      "loss": 0.0892,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.825603485107422,
      "rewards/margins": 13.686927795410156,
      "rewards/rejected": -21.512531280517578,
      "step": 680
    },
    {
      "epoch": 1.4322781525687598,
      "grad_norm": 16.75949934654318,
      "learning_rate": 1.1238402623334492e-07,
      "logits/chosen": -0.9597622752189636,
      "logits/rejected": 2.867584228515625,
      "logps/chosen": -1066.507568359375,
      "logps/rejected": -2423.34423828125,
      "loss": 0.0898,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -6.92177677154541,
      "rewards/margins": 13.205133438110352,
      "rewards/rejected": -20.126911163330078,
      "step": 690
    },
    {
      "epoch": 1.4530358069538143,
      "grad_norm": 11.338012307523448,
      "learning_rate": 1.0489613643176479e-07,
      "logits/chosen": -0.6678389310836792,
      "logits/rejected": 3.2197043895721436,
      "logps/chosen": -1113.594482421875,
      "logps/rejected": -2410.258544921875,
      "loss": 0.0722,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -7.322343349456787,
      "rewards/margins": 12.825822830200195,
      "rewards/rejected": -20.14816665649414,
      "step": 700
    },
    {
      "epoch": 1.4737934613388686,
      "grad_norm": 15.856495888079317,
      "learning_rate": 9.759962760723855e-08,
      "logits/chosen": -0.16049222648143768,
      "logits/rejected": 4.048083305358887,
      "logps/chosen": -1173.0849609375,
      "logps/rejected": -2468.91845703125,
      "loss": 0.0662,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -8.210199356079102,
      "rewards/margins": 12.80036735534668,
      "rewards/rejected": -21.01056671142578,
      "step": 710
    },
    {
      "epoch": 1.4945511157239233,
      "grad_norm": 31.07426456754591,
      "learning_rate": 9.050412330081883e-08,
      "logits/chosen": -0.20700784027576447,
      "logits/rejected": 3.9535317420959473,
      "logps/chosen": -1277.49755859375,
      "logps/rejected": -2737.533447265625,
      "loss": 0.0852,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -9.044589042663574,
      "rewards/margins": 14.331390380859375,
      "rewards/rejected": -23.375978469848633,
      "step": 720
    },
    {
      "epoch": 1.4945511157239233,
      "eval_logits/chosen": 0.06359091401100159,
      "eval_logits/rejected": 3.48725962638855,
      "eval_logps/chosen": -1424.8193359375,
      "eval_logps/rejected": -2816.588623046875,
      "eval_loss": 0.16444814205169678,
      "eval_rewards/accuracies": 0.9227941036224365,
      "eval_rewards/chosen": -10.01410961151123,
      "eval_rewards/margins": 13.69237232208252,
      "eval_rewards/rejected": -23.706483840942383,
      "eval_runtime": 297.3735,
      "eval_samples_per_second": 21.828,
      "eval_steps_per_second": 0.343,
      "step": 720
    },
    {
      "epoch": 1.5153087701089776,
      "grad_norm": 21.935108549149014,
      "learning_rate": 8.36189819438625e-08,
      "logits/chosen": -0.2683785557746887,
      "logits/rejected": 3.702204465866089,
      "logps/chosen": -1242.103759765625,
      "logps/rejected": -2603.80615234375,
      "loss": 0.0822,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -8.634117126464844,
      "rewards/margins": 13.532336235046387,
      "rewards/rejected": -22.166454315185547,
      "step": 730
    },
    {
      "epoch": 1.5360664244940323,
      "grad_norm": 23.902586381058878,
      "learning_rate": 7.69532845149711e-08,
      "logits/chosen": -0.5570476055145264,
      "logits/rejected": 3.649244785308838,
      "logps/chosen": -1223.75439453125,
      "logps/rejected": -2637.007080078125,
      "loss": 0.0769,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -8.374210357666016,
      "rewards/margins": 13.947656631469727,
      "rewards/rejected": -22.321866989135742,
      "step": 740
    },
    {
      "epoch": 1.5568240788790866,
      "grad_norm": 20.338315184147856,
      "learning_rate": 7.051582256286929e-08,
      "logits/chosen": -0.5207056999206543,
      "logits/rejected": 3.2942442893981934,
      "logps/chosen": -1216.895263671875,
      "logps/rejected": -2651.10107421875,
      "loss": 0.0706,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -8.260024070739746,
      "rewards/margins": 14.131828308105469,
      "rewards/rejected": -22.39185333251953,
      "step": 750
    },
    {
      "epoch": 1.5775817332641413,
      "grad_norm": 19.826108952738736,
      "learning_rate": 6.431508661101954e-08,
      "logits/chosen": -0.35470911860466003,
      "logits/rejected": 3.6188712120056152,
      "logps/chosen": -1201.515625,
      "logps/rejected": -2452.108642578125,
      "loss": 0.0906,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -8.28996467590332,
      "rewards/margins": 12.43807315826416,
      "rewards/rejected": -20.728038787841797,
      "step": 760
    },
    {
      "epoch": 1.5983393876491956,
      "grad_norm": 18.731154290431807,
      "learning_rate": 5.8359254959266826e-08,
      "logits/chosen": -0.3598397374153137,
      "logits/rejected": 3.461977481842041,
      "logps/chosen": -1123.501220703125,
      "logps/rejected": -2401.59423828125,
      "loss": 0.0757,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -7.732508659362793,
      "rewards/margins": 12.514566421508789,
      "rewards/rejected": -20.247074127197266,
      "step": 770
    },
    {
      "epoch": 1.61909704203425,
      "grad_norm": 14.021094730035166,
      "learning_rate": 5.265618289728199e-08,
      "logits/chosen": -0.2899594306945801,
      "logits/rejected": 3.6835436820983887,
      "logps/chosen": -1112.9346923828125,
      "logps/rejected": -2395.99560546875,
      "loss": 0.0748,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -7.564324378967285,
      "rewards/margins": 12.517146110534668,
      "rewards/rejected": -20.081470489501953,
      "step": 780
    },
    {
      "epoch": 1.6398546964193046,
      "grad_norm": 15.755848122162748,
      "learning_rate": 4.721339234403121e-08,
      "logits/chosen": -0.4539187550544739,
      "logits/rejected": 3.4312031269073486,
      "logps/chosen": -1102.984375,
      "logps/rejected": -2424.76904296875,
      "loss": 0.0889,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -7.416151523590088,
      "rewards/margins": 12.970865249633789,
      "rewards/rejected": -20.387014389038086,
      "step": 790
    },
    {
      "epoch": 1.660612350804359,
      "grad_norm": 10.116183369840204,
      "learning_rate": 4.203806192693587e-08,
      "logits/chosen": -0.16934530436992645,
      "logits/rejected": 3.3178775310516357,
      "logps/chosen": -1117.904541015625,
      "logps/rejected": -2234.350341796875,
      "loss": 0.0806,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -7.489673614501953,
      "rewards/margins": 10.942729949951172,
      "rewards/rejected": -18.432403564453125,
      "step": 800
    },
    {
      "epoch": 1.6813700051894136,
      "grad_norm": 12.314909116730863,
      "learning_rate": 3.7137017513808544e-08,
      "logits/chosen": -0.2919425368309021,
      "logits/rejected": 3.503552198410034,
      "logps/chosen": -1133.6949462890625,
      "logps/rejected": -2370.295166015625,
      "loss": 0.0813,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -7.621180057525635,
      "rewards/margins": 12.205830574035645,
      "rewards/rejected": -19.827011108398438,
      "step": 810
    },
    {
      "epoch": 1.702127659574468,
      "grad_norm": 13.100920531092996,
      "learning_rate": 3.251672321005147e-08,
      "logits/chosen": -0.3816925585269928,
      "logits/rejected": 3.223738193511963,
      "logps/chosen": -1128.720458984375,
      "logps/rejected": -2432.7451171875,
      "loss": 0.0799,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.558645725250244,
      "rewards/margins": 12.71537971496582,
      "rewards/rejected": -20.274024963378906,
      "step": 820
    },
    {
      "epoch": 1.7228853139595226,
      "grad_norm": 13.542985324247502,
      "learning_rate": 2.8183272832992267e-08,
      "logits/chosen": -0.4070394039154053,
      "logits/rejected": 3.100729465484619,
      "logps/chosen": -1098.7197265625,
      "logps/rejected": -2404.990234375,
      "loss": 0.0769,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.4235520362854,
      "rewards/margins": 12.707655906677246,
      "rewards/rejected": -20.131206512451172,
      "step": 830
    },
    {
      "epoch": 1.743642968344577,
      "grad_norm": 19.414761044152183,
      "learning_rate": 2.414238187460191e-08,
      "logits/chosen": -0.3680698275566101,
      "logits/rejected": 3.6575589179992676,
      "logps/chosen": -1149.003662109375,
      "logps/rejected": -2435.11474609375,
      "loss": 0.0801,
      "rewards/accuracies": 0.971875011920929,
      "rewards/chosen": -7.7628068923950195,
      "rewards/margins": 12.7561674118042,
      "rewards/rejected": -20.51897621154785,
      "step": 840
    },
    {
      "epoch": 1.7644006227296316,
      "grad_norm": 18.249543754589453,
      "learning_rate": 2.0399379963194713e-08,
      "logits/chosen": -0.15194618701934814,
      "logits/rejected": 3.823453903198242,
      "logps/chosen": -1178.7374267578125,
      "logps/rejected": -2647.753173828125,
      "loss": 0.0742,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -8.07176685333252,
      "rewards/margins": 14.458259582519531,
      "rewards/rejected": -22.530025482177734,
      "step": 850
    },
    {
      "epoch": 1.7851582771146859,
      "grad_norm": 13.687621613308757,
      "learning_rate": 1.695920383405322e-08,
      "logits/chosen": -0.1904297024011612,
      "logits/rejected": 3.487968921661377,
      "logps/chosen": -1222.4801025390625,
      "logps/rejected": -2662.45654296875,
      "loss": 0.0862,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.289772033691406,
      "rewards/margins": 14.194554328918457,
      "rewards/rejected": -22.484325408935547,
      "step": 860
    },
    {
      "epoch": 1.8059159314997406,
      "grad_norm": 20.833924751057065,
      "learning_rate": 1.3826390818249434e-08,
      "logits/chosen": -0.10482398420572281,
      "logits/rejected": 3.5555343627929688,
      "logps/chosen": -1184.642822265625,
      "logps/rejected": -2578.11376953125,
      "loss": 0.0847,
      "rewards/accuracies": 0.965624988079071,
      "rewards/chosen": -8.188472747802734,
      "rewards/margins": 13.555729866027832,
      "rewards/rejected": -21.74420166015625,
      "step": 870
    },
    {
      "epoch": 1.826673585884795,
      "grad_norm": 12.098887002832749,
      "learning_rate": 1.1005072858249614e-08,
      "logits/chosen": -0.21788470447063446,
      "logits/rejected": 3.596818447113037,
      "logps/chosen": -1198.203125,
      "logps/rejected": -2579.16455078125,
      "loss": 0.0742,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -8.193432807922363,
      "rewards/margins": 13.623662948608398,
      "rewards/rejected": -21.817094802856445,
      "step": 880
    },
    {
      "epoch": 1.8474312402698496,
      "grad_norm": 22.649648963902354,
      "learning_rate": 8.498971058195886e-09,
      "logits/chosen": -0.2010045051574707,
      "logits/rejected": 3.6617989540100098,
      "logps/chosen": -1205.736328125,
      "logps/rejected": -2611.108642578125,
      "loss": 0.0685,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -8.295499801635742,
      "rewards/margins": 13.880526542663574,
      "rewards/rejected": -22.176025390625,
      "step": 890
    },
    {
      "epoch": 1.868188894654904,
      "grad_norm": 14.001123095428785,
      "learning_rate": 6.311390776052527e-09,
      "logits/chosen": -0.26746782660484314,
      "logits/rejected": 3.480095386505127,
      "logps/chosen": -1162.368408203125,
      "logps/rejected": -2528.66650390625,
      "loss": 0.0735,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.984139442443848,
      "rewards/margins": 13.350987434387207,
      "rewards/rejected": -21.335126876831055,
      "step": 900
    },
    {
      "epoch": 1.8889465490399586,
      "grad_norm": 17.75181453766059,
      "learning_rate": 4.445217264089751e-09,
      "logits/chosen": -0.3570159077644348,
      "logits/rejected": 3.7746682167053223,
      "logps/chosen": -1153.1602783203125,
      "logps/rejected": -2572.643798828125,
      "loss": 0.0662,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.910887718200684,
      "rewards/margins": 13.993339538574219,
      "rewards/rejected": -21.904226303100586,
      "step": 910
    },
    {
      "epoch": 1.909704203425013,
      "grad_norm": 19.779224657698173,
      "learning_rate": 2.902911863455121e-09,
      "logits/chosen": -0.23463687300682068,
      "logits/rejected": 3.8247523307800293,
      "logps/chosen": -1181.158447265625,
      "logps/rejected": -2677.92041015625,
      "loss": 0.0761,
      "rewards/accuracies": 0.9781249761581421,
      "rewards/chosen": -8.19698429107666,
      "rewards/margins": 14.724853515625,
      "rewards/rejected": -22.92184066772461,
      "step": 920
    },
    {
      "epoch": 1.9304618578100676,
      "grad_norm": 22.12956855667942,
      "learning_rate": 1.686508757851507e-09,
      "logits/chosen": -0.2650212049484253,
      "logits/rejected": 3.5063083171844482,
      "logps/chosen": -1194.435791015625,
      "logps/rejected": -2615.01708984375,
      "loss": 0.0854,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -8.247761726379395,
      "rewards/margins": 13.95422077178955,
      "rewards/rejected": -22.201980590820312,
      "step": 930
    },
    {
      "epoch": 1.951219512195122,
      "grad_norm": 20.722879655487088,
      "learning_rate": 7.976122906031557e-10,
      "logits/chosen": -0.23710790276527405,
      "logits/rejected": 3.5924439430236816,
      "logps/chosen": -1181.043212890625,
      "logps/rejected": -2523.64794921875,
      "loss": 0.088,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -8.123555183410645,
      "rewards/margins": 13.19268798828125,
      "rewards/rejected": -21.31624412536621,
      "step": 940
    },
    {
      "epoch": 1.9719771665801764,
      "grad_norm": 12.897702624327897,
      "learning_rate": 2.37394848648792e-10,
      "logits/chosen": -0.3340440094470978,
      "logits/rejected": 3.253843307495117,
      "logps/chosen": -1149.8099365234375,
      "logps/rejected": -2490.43994140625,
      "loss": 0.0801,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.880959510803223,
      "rewards/margins": 13.208969116210938,
      "rewards/rejected": -21.089927673339844,
      "step": 950
    },
    {
      "epoch": 1.992734820965231,
      "grad_norm": 17.31058461781814,
      "learning_rate": 6.5953162521614755e-12,
      "logits/chosen": -0.3153567910194397,
      "logits/rejected": 3.3712565898895264,
      "logps/chosen": -1199.875244140625,
      "logps/rejected": -2640.38525390625,
      "loss": 0.0743,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -8.392365455627441,
      "rewards/margins": 14.206585884094238,
      "rewards/rejected": -22.598949432373047,
      "step": 960
    },
    {
      "epoch": 1.992734820965231,
      "eval_logits/chosen": -0.1724339723587036,
      "eval_logits/rejected": 3.016493320465088,
      "eval_logps/chosen": -1310.28955078125,
      "eval_logps/rejected": -2616.585205078125,
      "eval_loss": 0.15784408152103424,
      "eval_rewards/accuracies": 0.9203431606292725,
      "eval_rewards/chosen": -8.868812561035156,
      "eval_rewards/margins": 12.837637901306152,
      "eval_rewards/rejected": -21.706451416015625,
      "eval_runtime": 296.2637,
      "eval_samples_per_second": 21.91,
      "eval_steps_per_second": 0.344,
      "step": 960
    },
    {
      "epoch": 1.996886351842242,
      "step": 962,
      "total_flos": 0.0,
      "train_loss": 0.1585804910288655,
      "train_runtime": 28225.2793,
      "train_samples_per_second": 8.737,
      "train_steps_per_second": 0.034
    }
  ],
  "logging_steps": 10,
  "max_steps": 962,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 240,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}