{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.999297541394882,
  "eval_steps": 400,
  "global_step": 5604,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002676032781401572,
      "grad_norm": 5.052942243062624,
      "learning_rate": 8.9126559714795e-09,
      "logits/chosen": -0.06865417957305908,
      "logits/rejected": 0.14142072200775146,
      "logps/chosen": -1.7158018350601196,
      "logps/rejected": -1.8890635967254639,
      "loss": 0.6062,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.7158018350601196,
      "rewards/margins": 0.1732618361711502,
      "rewards/rejected": -1.8890635967254639,
      "sft_loss": 1.468388319015503,
      "step": 5
    },
    {
      "epoch": 0.005352065562803144,
      "grad_norm": 10.14750409435783,
      "learning_rate": 1.7825311942959e-08,
      "logits/chosen": -0.0027637421153485775,
      "logits/rejected": 0.11996668577194214,
      "logps/chosen": -1.8019473552703857,
      "logps/rejected": -1.8459587097167969,
      "loss": 0.6896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8019473552703857,
      "rewards/margins": 0.044011637568473816,
      "rewards/rejected": -1.8459587097167969,
      "sft_loss": 1.5082201957702637,
      "step": 10
    },
    {
      "epoch": 0.008028098344204716,
      "grad_norm": 11.385607782732158,
      "learning_rate": 2.67379679144385e-08,
      "logits/chosen": -0.041698895394802094,
      "logits/rejected": 0.05798926204442978,
      "logps/chosen": -1.6351089477539062,
      "logps/rejected": -1.7640068531036377,
      "loss": 0.6533,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.6351089477539062,
      "rewards/margins": 0.12889784574508667,
      "rewards/rejected": -1.7640068531036377,
      "sft_loss": 1.5005595684051514,
      "step": 15
    },
    {
      "epoch": 0.010704131125606288,
      "grad_norm": 5.148407815704086,
      "learning_rate": 3.5650623885918e-08,
      "logits/chosen": -0.04344125837087631,
      "logits/rejected": 0.04649763181805611,
      "logps/chosen": -1.7256911993026733,
      "logps/rejected": -1.8059346675872803,
      "loss": 0.6809,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.7256911993026733,
      "rewards/margins": 0.08024374395608902,
      "rewards/rejected": -1.8059346675872803,
      "sft_loss": 1.500503659248352,
      "step": 20
    },
    {
      "epoch": 0.013380163907007862,
      "grad_norm": 16.345046525598722,
      "learning_rate": 4.45632798573975e-08,
      "logits/chosen": -0.05768662691116333,
      "logits/rejected": 0.02871532365679741,
      "logps/chosen": -1.8689944744110107,
      "logps/rejected": -1.7781450748443604,
      "loss": 0.7893,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -1.8689944744110107,
      "rewards/margins": -0.09084945917129517,
      "rewards/rejected": -1.7781450748443604,
      "sft_loss": 1.5453780889511108,
      "step": 25
    },
    {
      "epoch": 0.016056196688409432,
      "grad_norm": 9.011620573094095,
      "learning_rate": 5.3475935828877e-08,
      "logits/chosen": -0.08053458482027054,
      "logits/rejected": 0.014007637277245522,
      "logps/chosen": -1.907224416732788,
      "logps/rejected": -1.8314783573150635,
      "loss": 0.7497,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -1.907224416732788,
      "rewards/margins": -0.07574623078107834,
      "rewards/rejected": -1.8314783573150635,
      "sft_loss": 1.645878791809082,
      "step": 30
    },
    {
      "epoch": 0.018732229469811006,
      "grad_norm": 10.780963310335471,
      "learning_rate": 6.23885918003565e-08,
      "logits/chosen": -0.049357324838638306,
      "logits/rejected": 0.11556194722652435,
      "logps/chosen": -1.8474754095077515,
      "logps/rejected": -1.9975019693374634,
      "loss": 0.7166,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.8474754095077515,
      "rewards/margins": 0.1500265896320343,
      "rewards/rejected": -1.9975019693374634,
      "sft_loss": 1.5622758865356445,
      "step": 35
    },
    {
      "epoch": 0.021408262251212576,
      "grad_norm": 10.010044826292075,
      "learning_rate": 7.1301247771836e-08,
      "logits/chosen": 0.04069102555513382,
      "logits/rejected": 0.21982774138450623,
      "logps/chosen": -1.8808033466339111,
      "logps/rejected": -1.7428737878799438,
      "loss": 0.7596,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -1.8808033466339111,
      "rewards/margins": -0.1379295140504837,
      "rewards/rejected": -1.7428737878799438,
      "sft_loss": 1.5191489458084106,
      "step": 40
    },
    {
      "epoch": 0.02408429503261415,
      "grad_norm": 15.791004960924981,
      "learning_rate": 8.021390374331551e-08,
      "logits/chosen": 0.015349939465522766,
      "logits/rejected": 0.2134595811367035,
      "logps/chosen": -1.835538625717163,
      "logps/rejected": -1.8691489696502686,
      "loss": 0.7282,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.835538625717163,
      "rewards/margins": 0.03361044079065323,
      "rewards/rejected": -1.8691489696502686,
      "sft_loss": 1.535658359527588,
      "step": 45
    },
    {
      "epoch": 0.026760327814015723,
      "grad_norm": 12.493095326259008,
      "learning_rate": 8.9126559714795e-08,
      "logits/chosen": -0.06104832887649536,
      "logits/rejected": 0.09160584956407547,
      "logps/chosen": -1.8953063488006592,
      "logps/rejected": -1.7769196033477783,
      "loss": 0.7798,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.8953063488006592,
      "rewards/margins": -0.11838690936565399,
      "rewards/rejected": -1.7769196033477783,
      "sft_loss": 1.5821568965911865,
      "step": 50
    },
    {
      "epoch": 0.029436360595417294,
      "grad_norm": 8.057579062835789,
      "learning_rate": 9.80392156862745e-08,
      "logits/chosen": -0.10174760967493057,
      "logits/rejected": 0.12272598594427109,
      "logps/chosen": -1.8287811279296875,
      "logps/rejected": -1.8614591360092163,
      "loss": 0.752,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8287811279296875,
      "rewards/margins": 0.032678090035915375,
      "rewards/rejected": -1.8614591360092163,
      "sft_loss": 1.5816882848739624,
      "step": 55
    },
    {
      "epoch": 0.032112393376818864,
      "grad_norm": 7.542081099548166,
      "learning_rate": 1.06951871657754e-07,
      "logits/chosen": -0.08671236038208008,
      "logits/rejected": 0.10760108381509781,
      "logps/chosen": -1.7833054065704346,
      "logps/rejected": -1.8888471126556396,
      "loss": 0.6705,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.7833054065704346,
      "rewards/margins": 0.10554170608520508,
      "rewards/rejected": -1.8888471126556396,
      "sft_loss": 1.5421648025512695,
      "step": 60
    },
    {
      "epoch": 0.03478842615822044,
      "grad_norm": 6.531349763687479,
      "learning_rate": 1.158645276292335e-07,
      "logits/chosen": -0.04039953276515007,
      "logits/rejected": 0.10648614168167114,
      "logps/chosen": -1.6316719055175781,
      "logps/rejected": -1.760345458984375,
      "loss": 0.6193,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.6316719055175781,
      "rewards/margins": 0.1286735087633133,
      "rewards/rejected": -1.760345458984375,
      "sft_loss": 1.4724668264389038,
      "step": 65
    },
    {
      "epoch": 0.03746445893962201,
      "grad_norm": 12.333828845131993,
      "learning_rate": 1.24777183600713e-07,
      "logits/chosen": -0.07198110967874527,
      "logits/rejected": 0.08535166829824448,
      "logps/chosen": -1.7603833675384521,
      "logps/rejected": -1.805906891822815,
      "loss": 0.7312,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -1.7603833675384521,
      "rewards/margins": 0.04552330821752548,
      "rewards/rejected": -1.805906891822815,
      "sft_loss": 1.6264307498931885,
      "step": 70
    },
    {
      "epoch": 0.04014049172102358,
      "grad_norm": 13.822532885111928,
      "learning_rate": 1.3368983957219251e-07,
      "logits/chosen": -0.055731602013111115,
      "logits/rejected": 0.1283172070980072,
      "logps/chosen": -1.7673460245132446,
      "logps/rejected": -2.0253078937530518,
      "loss": 0.6279,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.7673460245132446,
      "rewards/margins": 0.2579617202281952,
      "rewards/rejected": -2.0253078937530518,
      "sft_loss": 1.561607003211975,
      "step": 75
    },
    {
      "epoch": 0.04281652450242515,
      "grad_norm": 8.906433417273297,
      "learning_rate": 1.42602495543672e-07,
      "logits/chosen": 0.009920082986354828,
      "logits/rejected": 0.11612298339605331,
      "logps/chosen": -1.6991240978240967,
      "logps/rejected": -1.7324743270874023,
      "loss": 0.682,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.6991240978240967,
      "rewards/margins": 0.033349912613630295,
      "rewards/rejected": -1.7324743270874023,
      "sft_loss": 1.51686692237854,
      "step": 80
    },
    {
      "epoch": 0.04549255728382673,
      "grad_norm": 5.494631905202502,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -0.14454609155654907,
      "logits/rejected": 0.1083751767873764,
      "logps/chosen": -1.7641146183013916,
      "logps/rejected": -1.933570146560669,
      "loss": 0.65,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.7641146183013916,
      "rewards/margins": 0.16945549845695496,
      "rewards/rejected": -1.933570146560669,
      "sft_loss": 1.4863570928573608,
      "step": 85
    },
    {
      "epoch": 0.0481685900652283,
      "grad_norm": 15.69779541487828,
      "learning_rate": 1.6042780748663102e-07,
      "logits/chosen": 0.049393534660339355,
      "logits/rejected": 0.011763748712837696,
      "logps/chosen": -1.7164913415908813,
      "logps/rejected": -1.7509660720825195,
      "loss": 0.7072,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -1.7164913415908813,
      "rewards/margins": 0.03447474539279938,
      "rewards/rejected": -1.7509660720825195,
      "sft_loss": 1.4483808279037476,
      "step": 90
    },
    {
      "epoch": 0.05084462284662987,
      "grad_norm": 6.508984995318275,
      "learning_rate": 1.693404634581105e-07,
      "logits/chosen": -0.07589919865131378,
      "logits/rejected": 0.07544967532157898,
      "logps/chosen": -1.7599709033966064,
      "logps/rejected": -1.8785327672958374,
      "loss": 0.6772,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.7599709033966064,
      "rewards/margins": 0.11856192350387573,
      "rewards/rejected": -1.8785327672958374,
      "sft_loss": 1.5096004009246826,
      "step": 95
    },
    {
      "epoch": 0.05352065562803145,
      "grad_norm": 4.6447378329194215,
      "learning_rate": 1.7825311942959e-07,
      "logits/chosen": -0.03355532884597778,
      "logits/rejected": 0.03286678344011307,
      "logps/chosen": -1.6630899906158447,
      "logps/rejected": -1.7689199447631836,
      "loss": 0.642,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.6630899906158447,
      "rewards/margins": 0.10582991689443588,
      "rewards/rejected": -1.7689199447631836,
      "sft_loss": 1.4793955087661743,
      "step": 100
    },
    {
      "epoch": 0.05619668840943302,
      "grad_norm": 10.397721008956594,
      "learning_rate": 1.8716577540106952e-07,
      "logits/chosen": 0.04035579040646553,
      "logits/rejected": 0.067632295191288,
      "logps/chosen": -1.6082446575164795,
      "logps/rejected": -1.7760941982269287,
      "loss": 0.6146,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.6082446575164795,
      "rewards/margins": 0.16784964501857758,
      "rewards/rejected": -1.7760941982269287,
      "sft_loss": 1.4214261770248413,
      "step": 105
    },
    {
      "epoch": 0.05887272119083459,
      "grad_norm": 6.249475901409631,
      "learning_rate": 1.96078431372549e-07,
      "logits/chosen": 0.0038494463078677654,
      "logits/rejected": 0.1005120500922203,
      "logps/chosen": -1.6075687408447266,
      "logps/rejected": -1.6648200750350952,
      "loss": 0.6582,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6075687408447266,
      "rewards/margins": 0.0572517029941082,
      "rewards/rejected": -1.6648200750350952,
      "sft_loss": 1.4360030889511108,
      "step": 110
    },
    {
      "epoch": 0.06154875397223616,
      "grad_norm": 10.847905194112093,
      "learning_rate": 2.049910873440285e-07,
      "logits/chosen": 0.034572064876556396,
      "logits/rejected": 0.2488282471895218,
      "logps/chosen": -1.5851459503173828,
      "logps/rejected": -1.8454182147979736,
      "loss": 0.5819,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5851459503173828,
      "rewards/margins": 0.26027244329452515,
      "rewards/rejected": -1.8454182147979736,
      "sft_loss": 1.52280855178833,
      "step": 115
    },
    {
      "epoch": 0.06422478675363773,
      "grad_norm": 6.366957134835353,
      "learning_rate": 2.13903743315508e-07,
      "logits/chosen": -0.0667978897690773,
      "logits/rejected": 0.11292694509029388,
      "logps/chosen": -1.6360286474227905,
      "logps/rejected": -1.7481434345245361,
      "loss": 0.6288,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6360286474227905,
      "rewards/margins": 0.11211474239826202,
      "rewards/rejected": -1.7481434345245361,
      "sft_loss": 1.5068069696426392,
      "step": 120
    },
    {
      "epoch": 0.0669008195350393,
      "grad_norm": 4.843145580690555,
      "learning_rate": 2.2281639928698751e-07,
      "logits/chosen": -0.08813653141260147,
      "logits/rejected": 0.044660698622465134,
      "logps/chosen": -1.5761592388153076,
      "logps/rejected": -1.541815996170044,
      "loss": 0.672,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -1.5761592388153076,
      "rewards/margins": -0.034343421459198,
      "rewards/rejected": -1.541815996170044,
      "sft_loss": 1.4881436824798584,
      "step": 125
    },
    {
      "epoch": 0.06957685231644088,
      "grad_norm": 8.927002657528016,
      "learning_rate": 2.31729055258467e-07,
      "logits/chosen": 0.009452106431126595,
      "logits/rejected": 0.1407095491886139,
      "logps/chosen": -1.6128228902816772,
      "logps/rejected": -1.730139970779419,
      "loss": 0.5943,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.6128228902816772,
      "rewards/margins": 0.11731685698032379,
      "rewards/rejected": -1.730139970779419,
      "sft_loss": 1.5444576740264893,
      "step": 130
    },
    {
      "epoch": 0.07225288509784245,
      "grad_norm": 16.0318675129108,
      "learning_rate": 2.406417112299465e-07,
      "logits/chosen": -0.07095210254192352,
      "logits/rejected": 0.04487922415137291,
      "logps/chosen": -1.6604677438735962,
      "logps/rejected": -1.6921546459197998,
      "loss": 0.6703,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.6604677438735962,
      "rewards/margins": 0.0316871777176857,
      "rewards/rejected": -1.6921546459197998,
      "sft_loss": 1.4799906015396118,
      "step": 135
    },
    {
      "epoch": 0.07492891787924402,
      "grad_norm": 7.8260263801664,
      "learning_rate": 2.49554367201426e-07,
      "logits/chosen": -0.05099118873476982,
      "logits/rejected": 0.11860334873199463,
      "logps/chosen": -1.6225074529647827,
      "logps/rejected": -1.741729497909546,
      "loss": 0.6204,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.6225074529647827,
      "rewards/margins": 0.11922208964824677,
      "rewards/rejected": -1.741729497909546,
      "sft_loss": 1.5272603034973145,
      "step": 140
    },
    {
      "epoch": 0.0776049506606456,
      "grad_norm": 10.071220149350514,
      "learning_rate": 2.5846702317290554e-07,
      "logits/chosen": -0.02638971246778965,
      "logits/rejected": 0.127262145280838,
      "logps/chosen": -1.528851866722107,
      "logps/rejected": -1.635650873184204,
      "loss": 0.6073,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.528851866722107,
      "rewards/margins": 0.10679880529642105,
      "rewards/rejected": -1.635650873184204,
      "sft_loss": 1.4772685766220093,
      "step": 145
    },
    {
      "epoch": 0.08028098344204716,
      "grad_norm": 12.863350569849993,
      "learning_rate": 2.6737967914438503e-07,
      "logits/chosen": -0.08558139950037003,
      "logits/rejected": 0.07510876655578613,
      "logps/chosen": -1.4721580743789673,
      "logps/rejected": -1.472564935684204,
      "loss": 0.64,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4721580743789673,
      "rewards/margins": 0.000406736129662022,
      "rewards/rejected": -1.472564935684204,
      "sft_loss": 1.3356287479400635,
      "step": 150
    },
    {
      "epoch": 0.08295701622344874,
      "grad_norm": 8.630736310404208,
      "learning_rate": 2.762923351158645e-07,
      "logits/chosen": -0.07413525134325027,
      "logits/rejected": -0.02334105595946312,
      "logps/chosen": -1.4584012031555176,
      "logps/rejected": -1.562864065170288,
      "loss": 0.5902,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.4584012031555176,
      "rewards/margins": 0.10446293652057648,
      "rewards/rejected": -1.562864065170288,
      "sft_loss": 1.4039736986160278,
      "step": 155
    },
    {
      "epoch": 0.0856330490048503,
      "grad_norm": 8.976834922671944,
      "learning_rate": 2.85204991087344e-07,
      "logits/chosen": -0.18383175134658813,
      "logits/rejected": -0.04810567945241928,
      "logps/chosen": -1.522956371307373,
      "logps/rejected": -1.5014146566390991,
      "loss": 0.661,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.522956371307373,
      "rewards/margins": -0.021541709080338478,
      "rewards/rejected": -1.5014146566390991,
      "sft_loss": 1.433183193206787,
      "step": 160
    },
    {
      "epoch": 0.08830908178625188,
      "grad_norm": 7.445472741165519,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -0.10111711174249649,
      "logits/rejected": 0.06512321531772614,
      "logps/chosen": -1.3696064949035645,
      "logps/rejected": -1.4571757316589355,
      "loss": 0.595,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.3696064949035645,
      "rewards/margins": 0.08756937086582184,
      "rewards/rejected": -1.4571757316589355,
      "sft_loss": 1.324439287185669,
      "step": 165
    },
    {
      "epoch": 0.09098511456765346,
      "grad_norm": 14.656981338791415,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -0.11693856865167618,
      "logits/rejected": -0.06399437040090561,
      "logps/chosen": -1.4670077562332153,
      "logps/rejected": -1.536694049835205,
      "loss": 0.6139,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.4670077562332153,
      "rewards/margins": 0.06968621164560318,
      "rewards/rejected": -1.536694049835205,
      "sft_loss": 1.440754771232605,
      "step": 170
    },
    {
      "epoch": 0.09366114734905502,
      "grad_norm": 6.584009251136716,
      "learning_rate": 3.1194295900178254e-07,
      "logits/chosen": -0.006804236676543951,
      "logits/rejected": -0.009001683443784714,
      "logps/chosen": -1.3459522724151611,
      "logps/rejected": -1.4413591623306274,
      "loss": 0.5751,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.3459522724151611,
      "rewards/margins": 0.09540703147649765,
      "rewards/rejected": -1.4413591623306274,
      "sft_loss": 1.368872046470642,
      "step": 175
    },
    {
      "epoch": 0.0963371801304566,
      "grad_norm": 6.038444404374204,
      "learning_rate": 3.2085561497326203e-07,
      "logits/chosen": -0.06775864958763123,
      "logits/rejected": -0.06924072653055191,
      "logps/chosen": -1.3591883182525635,
      "logps/rejected": -1.560471773147583,
      "loss": 0.5766,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3591883182525635,
      "rewards/margins": 0.20128345489501953,
      "rewards/rejected": -1.560471773147583,
      "sft_loss": 1.378900408744812,
      "step": 180
    },
    {
      "epoch": 0.09901321291185818,
      "grad_norm": 6.851019247142483,
      "learning_rate": 3.297682709447415e-07,
      "logits/chosen": -0.20338714122772217,
      "logits/rejected": -0.1196417361497879,
      "logps/chosen": -1.3481305837631226,
      "logps/rejected": -1.3971184492111206,
      "loss": 0.601,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.3481305837631226,
      "rewards/margins": 0.04898771643638611,
      "rewards/rejected": -1.3971184492111206,
      "sft_loss": 1.3574292659759521,
      "step": 185
    },
    {
      "epoch": 0.10168924569325974,
      "grad_norm": 6.199864509350299,
      "learning_rate": 3.38680926916221e-07,
      "logits/chosen": -0.11407258361577988,
      "logits/rejected": -0.0008642614120617509,
      "logps/chosen": -1.2794859409332275,
      "logps/rejected": -1.4109026193618774,
      "loss": 0.5487,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.2794859409332275,
      "rewards/margins": 0.13141660392284393,
      "rewards/rejected": -1.4109026193618774,
      "sft_loss": 1.2981491088867188,
      "step": 190
    },
    {
      "epoch": 0.10436527847466132,
      "grad_norm": 4.548236551688456,
      "learning_rate": 3.475935828877005e-07,
      "logits/chosen": -0.05843614414334297,
      "logits/rejected": 0.08690853416919708,
      "logps/chosen": -1.2500104904174805,
      "logps/rejected": -1.4155646562576294,
      "loss": 0.528,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.2500104904174805,
      "rewards/margins": 0.16555407643318176,
      "rewards/rejected": -1.4155646562576294,
      "sft_loss": 1.2920830249786377,
      "step": 195
    },
    {
      "epoch": 0.1070413112560629,
      "grad_norm": 15.793009061938657,
      "learning_rate": 3.5650623885918e-07,
      "logits/chosen": -0.15543696284294128,
      "logits/rejected": -0.02773769572377205,
      "logps/chosen": -1.3737095594406128,
      "logps/rejected": -1.413891077041626,
      "loss": 0.5936,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.3737095594406128,
      "rewards/margins": 0.04018155485391617,
      "rewards/rejected": -1.413891077041626,
      "sft_loss": 1.3922524452209473,
      "step": 200
    },
    {
      "epoch": 0.10971734403746446,
      "grad_norm": 10.870168642235853,
      "learning_rate": 3.654188948306595e-07,
      "logits/chosen": -0.0997370257973671,
      "logits/rejected": 0.03656370937824249,
      "logps/chosen": -1.2865569591522217,
      "logps/rejected": -1.3560245037078857,
      "loss": 0.5739,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.2865569591522217,
      "rewards/margins": 0.06946732848882675,
      "rewards/rejected": -1.3560245037078857,
      "sft_loss": 1.2838119268417358,
      "step": 205
    },
    {
      "epoch": 0.11239337681886603,
      "grad_norm": 7.032884322797753,
      "learning_rate": 3.7433155080213904e-07,
      "logits/chosen": -0.2042866200208664,
      "logits/rejected": -0.028805147856473923,
      "logps/chosen": -1.360367774963379,
      "logps/rejected": -1.473173975944519,
      "loss": 0.5842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.360367774963379,
      "rewards/margins": 0.11280622333288193,
      "rewards/rejected": -1.473173975944519,
      "sft_loss": 1.3383769989013672,
      "step": 210
    },
    {
      "epoch": 0.1150694096002676,
      "grad_norm": 5.142657637211093,
      "learning_rate": 3.8324420677361853e-07,
      "logits/chosen": -0.22696277499198914,
      "logits/rejected": 0.007920200005173683,
      "logps/chosen": -1.3814806938171387,
      "logps/rejected": -1.436943769454956,
      "loss": 0.5806,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3814806938171387,
      "rewards/margins": 0.05546308681368828,
      "rewards/rejected": -1.436943769454956,
      "sft_loss": 1.3624300956726074,
      "step": 215
    },
    {
      "epoch": 0.11774544238166917,
      "grad_norm": 13.237933869494071,
      "learning_rate": 3.92156862745098e-07,
      "logits/chosen": -0.008888229727745056,
      "logits/rejected": 0.08174064010381699,
      "logps/chosen": -1.306950569152832,
      "logps/rejected": -1.4514024257659912,
      "loss": 0.561,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.306950569152832,
      "rewards/margins": 0.14445172250270844,
      "rewards/rejected": -1.4514024257659912,
      "sft_loss": 1.3328325748443604,
      "step": 220
    },
    {
      "epoch": 0.12042147516307075,
      "grad_norm": 4.585259904611552,
      "learning_rate": 4.010695187165775e-07,
      "logits/chosen": -0.15518362820148468,
      "logits/rejected": 0.0046308608725667,
      "logps/chosen": -1.3100550174713135,
      "logps/rejected": -1.4391072988510132,
      "loss": 0.5474,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3100550174713135,
      "rewards/margins": 0.12905225157737732,
      "rewards/rejected": -1.4391072988510132,
      "sft_loss": 1.3125994205474854,
      "step": 225
    },
    {
      "epoch": 0.12309750794447231,
      "grad_norm": 4.796634954673317,
      "learning_rate": 4.09982174688057e-07,
      "logits/chosen": -0.06799010187387466,
      "logits/rejected": 0.005717620253562927,
      "logps/chosen": -1.3139328956604004,
      "logps/rejected": -1.4714056253433228,
      "loss": 0.5433,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3139328956604004,
      "rewards/margins": 0.15747258067131042,
      "rewards/rejected": -1.4714056253433228,
      "sft_loss": 1.2739067077636719,
      "step": 230
    },
    {
      "epoch": 0.1257735407258739,
      "grad_norm": 6.068497866740446,
      "learning_rate": 4.188948306595365e-07,
      "logits/chosen": -0.037981465458869934,
      "logits/rejected": 0.08764432370662689,
      "logps/chosen": -1.2823946475982666,
      "logps/rejected": -1.4488177299499512,
      "loss": 0.5325,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.2823946475982666,
      "rewards/margins": 0.16642294824123383,
      "rewards/rejected": -1.4488177299499512,
      "sft_loss": 1.2844218015670776,
      "step": 235
    },
    {
      "epoch": 0.12844957350727546,
      "grad_norm": 4.365912398384617,
      "learning_rate": 4.27807486631016e-07,
      "logits/chosen": -0.09100799262523651,
      "logits/rejected": 0.025594225153326988,
      "logps/chosen": -1.297890543937683,
      "logps/rejected": -1.481879711151123,
      "loss": 0.5342,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.297890543937683,
      "rewards/margins": 0.18398909270763397,
      "rewards/rejected": -1.481879711151123,
      "sft_loss": 1.3451718091964722,
      "step": 240
    },
    {
      "epoch": 0.13112560628867703,
      "grad_norm": 6.346243072159388,
      "learning_rate": 4.3672014260249554e-07,
      "logits/chosen": -0.007030495908111334,
      "logits/rejected": 0.10519347339868546,
      "logps/chosen": -1.4173331260681152,
      "logps/rejected": -1.44147527217865,
      "loss": 0.6128,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.4173331260681152,
      "rewards/margins": 0.024142302572727203,
      "rewards/rejected": -1.44147527217865,
      "sft_loss": 1.4315967559814453,
      "step": 245
    },
    {
      "epoch": 0.1338016390700786,
      "grad_norm": 8.178991051627037,
      "learning_rate": 4.4563279857397503e-07,
      "logits/chosen": -0.11276876926422119,
      "logits/rejected": 0.04048747569322586,
      "logps/chosen": -1.2980380058288574,
      "logps/rejected": -1.3559579849243164,
      "loss": 0.5891,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.2980380058288574,
      "rewards/margins": 0.05791999027132988,
      "rewards/rejected": -1.3559579849243164,
      "sft_loss": 1.3147931098937988,
      "step": 250
    },
    {
      "epoch": 0.1364776718514802,
      "grad_norm": 5.943792587616856,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -0.051166605204343796,
      "logits/rejected": 0.08564223349094391,
      "logps/chosen": -1.2638366222381592,
      "logps/rejected": -1.3699913024902344,
      "loss": 0.5457,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.2638366222381592,
      "rewards/margins": 0.106154665350914,
      "rewards/rejected": -1.3699913024902344,
      "sft_loss": 1.2442152500152588,
      "step": 255
    },
    {
      "epoch": 0.13915370463288176,
      "grad_norm": 5.011172706823575,
      "learning_rate": 4.63458110516934e-07,
      "logits/chosen": -0.26364797353744507,
      "logits/rejected": -0.1617596596479416,
      "logps/chosen": -1.3498268127441406,
      "logps/rejected": -1.5086586475372314,
      "loss": 0.5425,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3498268127441406,
      "rewards/margins": 0.15883179008960724,
      "rewards/rejected": -1.5086586475372314,
      "sft_loss": 1.3699219226837158,
      "step": 260
    },
    {
      "epoch": 0.1418297374142833,
      "grad_norm": 5.581957559969404,
      "learning_rate": 4.723707664884135e-07,
      "logits/chosen": -0.11523625999689102,
      "logits/rejected": -0.029855186119675636,
      "logps/chosen": -1.3342697620391846,
      "logps/rejected": -1.5049277544021606,
      "loss": 0.5582,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.3342697620391846,
      "rewards/margins": 0.17065785825252533,
      "rewards/rejected": -1.5049277544021606,
      "sft_loss": 1.3940002918243408,
      "step": 265
    },
    {
      "epoch": 0.1445057701956849,
      "grad_norm": 3.8784079567677807,
      "learning_rate": 4.81283422459893e-07,
      "logits/chosen": -0.13867095112800598,
      "logits/rejected": -0.01858574151992798,
      "logps/chosen": -1.315878987312317,
      "logps/rejected": -1.4139220714569092,
      "loss": 0.5585,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.315878987312317,
      "rewards/margins": 0.09804315865039825,
      "rewards/rejected": -1.4139220714569092,
      "sft_loss": 1.334929347038269,
      "step": 270
    },
    {
      "epoch": 0.14718180297708647,
      "grad_norm": 4.914311906008871,
      "learning_rate": 4.901960784313725e-07,
      "logits/chosen": -0.06731410324573517,
      "logits/rejected": 0.025944000110030174,
      "logps/chosen": -1.2634482383728027,
      "logps/rejected": -1.4206976890563965,
      "loss": 0.5289,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.2634482383728027,
      "rewards/margins": 0.15724948048591614,
      "rewards/rejected": -1.4206976890563965,
      "sft_loss": 1.2450734376907349,
      "step": 275
    },
    {
      "epoch": 0.14985783575848804,
      "grad_norm": 5.175769918745935,
      "learning_rate": 4.99108734402852e-07,
      "logits/chosen": -0.13446274399757385,
      "logits/rejected": 0.018282266333699226,
      "logps/chosen": -1.3150831460952759,
      "logps/rejected": -1.414188027381897,
      "loss": 0.5669,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3150831460952759,
      "rewards/margins": 0.09910491853952408,
      "rewards/rejected": -1.414188027381897,
      "sft_loss": 1.3091659545898438,
      "step": 280
    },
    {
      "epoch": 0.15253386853988962,
      "grad_norm": 5.877826446213898,
      "learning_rate": 5.080213903743315e-07,
      "logits/chosen": -0.11841098964214325,
      "logits/rejected": 0.011662433855235577,
      "logps/chosen": -1.3436254262924194,
      "logps/rejected": -1.4265941381454468,
      "loss": 0.5827,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3436254262924194,
      "rewards/margins": 0.08296870440244675,
      "rewards/rejected": -1.4265941381454468,
      "sft_loss": 1.3912488222122192,
      "step": 285
    },
    {
      "epoch": 0.1552099013212912,
      "grad_norm": 5.694699355384885,
      "learning_rate": 5.169340463458111e-07,
      "logits/chosen": -0.14523978531360626,
      "logits/rejected": 0.14297033846378326,
      "logps/chosen": -1.3640129566192627,
      "logps/rejected": -1.48271644115448,
      "loss": 0.5624,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3640129566192627,
      "rewards/margins": 0.1187034398317337,
      "rewards/rejected": -1.48271644115448,
      "sft_loss": 1.3594133853912354,
      "step": 290
    },
    {
      "epoch": 0.15788593410269275,
      "grad_norm": 5.834241126679447,
      "learning_rate": 5.258467023172905e-07,
      "logits/chosen": -0.09991417825222015,
      "logits/rejected": -0.04301440715789795,
      "logps/chosen": -1.2645776271820068,
      "logps/rejected": -1.4023511409759521,
      "loss": 0.5371,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.2645776271820068,
      "rewards/margins": 0.13777343928813934,
      "rewards/rejected": -1.4023511409759521,
      "sft_loss": 1.2729285955429077,
      "step": 295
    },
    {
      "epoch": 0.16056196688409433,
      "grad_norm": 5.597124434517011,
      "learning_rate": 5.347593582887701e-07,
      "logits/chosen": -0.09893033653497696,
      "logits/rejected": 0.06294500082731247,
      "logps/chosen": -1.301417350769043,
      "logps/rejected": -1.3780789375305176,
      "loss": 0.5627,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.301417350769043,
      "rewards/margins": 0.07666166126728058,
      "rewards/rejected": -1.3780789375305176,
      "sft_loss": 1.3655966520309448,
      "step": 300
    },
    {
      "epoch": 0.1632379996654959,
      "grad_norm": 4.26674515151328,
      "learning_rate": 5.436720142602496e-07,
      "logits/chosen": -0.06475608050823212,
      "logits/rejected": 0.005105156451463699,
      "logps/chosen": -1.406415581703186,
      "logps/rejected": -1.4127743244171143,
      "loss": 0.6297,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.406415581703186,
      "rewards/margins": 0.006358677055686712,
      "rewards/rejected": -1.4127743244171143,
      "sft_loss": 1.410060167312622,
      "step": 305
    },
    {
      "epoch": 0.16591403244689748,
      "grad_norm": 6.211235277860888,
      "learning_rate": 5.52584670231729e-07,
      "logits/chosen": -0.2585568428039551,
      "logits/rejected": -0.1752055436372757,
      "logps/chosen": -1.3687752485275269,
      "logps/rejected": -1.4534339904785156,
      "loss": 0.5973,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.3687752485275269,
      "rewards/margins": 0.0846589207649231,
      "rewards/rejected": -1.4534339904785156,
      "sft_loss": 1.3693883419036865,
      "step": 310
    },
    {
      "epoch": 0.16859006522829906,
      "grad_norm": 6.564026888050156,
      "learning_rate": 5.614973262032086e-07,
      "logits/chosen": -0.04630298539996147,
      "logits/rejected": 0.1092165932059288,
      "logps/chosen": -1.3590734004974365,
      "logps/rejected": -1.5081905126571655,
      "loss": 0.5845,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.3590734004974365,
      "rewards/margins": 0.149117112159729,
      "rewards/rejected": -1.5081905126571655,
      "sft_loss": 1.3778035640716553,
      "step": 315
    },
    {
      "epoch": 0.1712660980097006,
      "grad_norm": 4.348469697914348,
      "learning_rate": 5.70409982174688e-07,
      "logits/chosen": -0.09974730014801025,
      "logits/rejected": 0.02833659015595913,
      "logps/chosen": -1.3171430826187134,
      "logps/rejected": -1.3695036172866821,
      "loss": 0.5721,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.3171430826187134,
      "rewards/margins": 0.052360523492097855,
      "rewards/rejected": -1.3695036172866821,
      "sft_loss": 1.339951992034912,
      "step": 320
    },
    {
      "epoch": 0.17394213079110218,
      "grad_norm": 5.105999096976248,
      "learning_rate": 5.793226381461676e-07,
      "logits/chosen": -0.16527916491031647,
      "logits/rejected": -0.05536722391843796,
      "logps/chosen": -1.3153146505355835,
      "logps/rejected": -1.5632933378219604,
      "loss": 0.5331,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3153146505355835,
      "rewards/margins": 0.24797877669334412,
      "rewards/rejected": -1.5632933378219604,
      "sft_loss": 1.4050966501235962,
      "step": 325
    },
    {
      "epoch": 0.17661816357250376,
      "grad_norm": 8.82419196921446,
      "learning_rate": 5.88235294117647e-07,
      "logits/chosen": -0.07711423933506012,
      "logits/rejected": 0.06198772042989731,
      "logps/chosen": -1.3323601484298706,
      "logps/rejected": -1.50588059425354,
      "loss": 0.55,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.3323601484298706,
      "rewards/margins": 0.17352035641670227,
      "rewards/rejected": -1.50588059425354,
      "sft_loss": 1.3518102169036865,
      "step": 330
    },
    {
      "epoch": 0.17929419635390534,
      "grad_norm": 8.09083744856097,
      "learning_rate": 5.971479500891266e-07,
      "logits/chosen": 0.0038772523403167725,
      "logits/rejected": 0.10497460514307022,
      "logps/chosen": -1.3427479267120361,
      "logps/rejected": -1.3789106607437134,
      "loss": 0.5807,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.3427479267120361,
      "rewards/margins": 0.036162860691547394,
      "rewards/rejected": -1.3789106607437134,
      "sft_loss": 1.3472321033477783,
      "step": 335
    },
    {
      "epoch": 0.18197022913530692,
      "grad_norm": 7.465959336338128,
      "learning_rate": 6.060606060606061e-07,
      "logits/chosen": -0.05871904641389847,
      "logits/rejected": 0.08265434950590134,
      "logps/chosen": -1.4018275737762451,
      "logps/rejected": -1.488533854484558,
      "loss": 0.5994,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.4018275737762451,
      "rewards/margins": 0.08670647442340851,
      "rewards/rejected": -1.488533854484558,
      "sft_loss": 1.3964767456054688,
      "step": 340
    },
    {
      "epoch": 0.1846462619167085,
      "grad_norm": 8.49429791242324,
      "learning_rate": 6.149732620320855e-07,
      "logits/chosen": 0.0036362111568450928,
      "logits/rejected": 0.03215277940034866,
      "logps/chosen": -1.3091847896575928,
      "logps/rejected": -1.4534891843795776,
      "loss": 0.5495,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3091847896575928,
      "rewards/margins": 0.1443043351173401,
      "rewards/rejected": -1.4534891843795776,
      "sft_loss": 1.3581584692001343,
      "step": 345
    },
    {
      "epoch": 0.18732229469811004,
      "grad_norm": 5.6762514137157165,
      "learning_rate": 6.238859180035651e-07,
      "logits/chosen": -0.04075011983513832,
      "logits/rejected": 0.05240970104932785,
      "logps/chosen": -1.2877025604248047,
      "logps/rejected": -1.4130481481552124,
      "loss": 0.5576,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.2877025604248047,
      "rewards/margins": 0.12534549832344055,
      "rewards/rejected": -1.4130481481552124,
      "sft_loss": 1.350273847579956,
      "step": 350
    },
    {
      "epoch": 0.18999832747951162,
      "grad_norm": 5.6317484450607225,
      "learning_rate": 6.327985739750445e-07,
      "logits/chosen": -0.14434251189231873,
      "logits/rejected": 0.06206267327070236,
      "logps/chosen": -1.3909976482391357,
      "logps/rejected": -1.4239745140075684,
      "loss": 0.6073,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -1.3909976482391357,
      "rewards/margins": 0.032976895570755005,
      "rewards/rejected": -1.4239745140075684,
      "sft_loss": 1.4065606594085693,
      "step": 355
    },
    {
      "epoch": 0.1926743602609132,
      "grad_norm": 5.730528263535874,
      "learning_rate": 6.417112299465241e-07,
      "logits/chosen": -0.10777083784341812,
      "logits/rejected": -0.0323818065226078,
      "logps/chosen": -1.3194472789764404,
      "logps/rejected": -1.4491716623306274,
      "loss": 0.5493,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3194472789764404,
      "rewards/margins": 0.12972450256347656,
      "rewards/rejected": -1.4491716623306274,
      "sft_loss": 1.3073797225952148,
      "step": 360
    },
    {
      "epoch": 0.19535039304231477,
      "grad_norm": 7.441457504970106,
      "learning_rate": 6.506238859180035e-07,
      "logits/chosen": -0.03163684159517288,
      "logits/rejected": 0.046002812683582306,
      "logps/chosen": -1.2995356321334839,
      "logps/rejected": -1.405275821685791,
      "loss": 0.5665,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.2995356321334839,
      "rewards/margins": 0.10574007034301758,
      "rewards/rejected": -1.405275821685791,
      "sft_loss": 1.2813327312469482,
      "step": 365
    },
    {
      "epoch": 0.19802642582371635,
      "grad_norm": 4.578902430418264,
      "learning_rate": 6.59536541889483e-07,
      "logits/chosen": -0.049534980207681656,
      "logits/rejected": 0.04207900911569595,
      "logps/chosen": -1.2937860488891602,
      "logps/rejected": -1.3419485092163086,
      "loss": 0.5784,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.2937860488891602,
      "rewards/margins": 0.048162512481212616,
      "rewards/rejected": -1.3419485092163086,
      "sft_loss": 1.2689062356948853,
      "step": 370
    },
    {
      "epoch": 0.2007024586051179,
      "grad_norm": 5.977001125118096,
      "learning_rate": 6.684491978609626e-07,
      "logits/chosen": -0.10349956899881363,
      "logits/rejected": 0.04922555759549141,
      "logps/chosen": -1.2733392715454102,
      "logps/rejected": -1.4207669496536255,
      "loss": 0.546,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.2733392715454102,
      "rewards/margins": 0.14742770791053772,
      "rewards/rejected": -1.4207669496536255,
      "sft_loss": 1.3344981670379639,
      "step": 375
    },
    {
      "epoch": 0.20337849138651948,
      "grad_norm": 4.7894362626255,
      "learning_rate": 6.77361853832442e-07,
      "logits/chosen": -0.05292842537164688,
      "logits/rejected": 0.030161788687109947,
      "logps/chosen": -1.2897326946258545,
      "logps/rejected": -1.4556329250335693,
      "loss": 0.5365,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.2897326946258545,
      "rewards/margins": 0.16590023040771484,
      "rewards/rejected": -1.4556329250335693,
      "sft_loss": 1.305248498916626,
      "step": 380
    },
    {
      "epoch": 0.20605452416792105,
      "grad_norm": 4.0595535401027645,
      "learning_rate": 6.862745098039216e-07,
      "logits/chosen": -0.0235292986035347,
      "logits/rejected": 0.05328557640314102,
      "logps/chosen": -1.3905055522918701,
      "logps/rejected": -1.3857368230819702,
      "loss": 0.6318,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.3905055522918701,
      "rewards/margins": -0.004768743179738522,
      "rewards/rejected": -1.3857368230819702,
      "sft_loss": 1.4026410579681396,
      "step": 385
    },
    {
      "epoch": 0.20873055694932263,
      "grad_norm": 7.727865438010638,
      "learning_rate": 6.95187165775401e-07,
      "logits/chosen": 0.042860690504312515,
      "logits/rejected": 0.2084215134382248,
      "logps/chosen": -1.383760690689087,
      "logps/rejected": -1.4487297534942627,
      "loss": 0.601,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -1.383760690689087,
      "rewards/margins": 0.06496912240982056,
      "rewards/rejected": -1.4487297534942627,
      "sft_loss": 1.392791748046875,
      "step": 390
    },
    {
      "epoch": 0.2114065897307242,
      "grad_norm": 5.87319636795186,
      "learning_rate": 7.040998217468806e-07,
      "logits/chosen": -0.07385215908288956,
      "logits/rejected": 0.08165531605482101,
      "logps/chosen": -1.3311251401901245,
      "logps/rejected": -1.3491557836532593,
      "loss": 0.5836,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.3311251401901245,
      "rewards/margins": 0.01803075149655342,
      "rewards/rejected": -1.3491557836532593,
      "sft_loss": 1.3450191020965576,
      "step": 395
    },
    {
      "epoch": 0.2140826225121258,
      "grad_norm": 4.475996032871945,
      "learning_rate": 7.1301247771836e-07,
      "logits/chosen": 0.04886271804571152,
      "logits/rejected": 0.1430073380470276,
      "logps/chosen": -1.3099935054779053,
      "logps/rejected": -1.4135615825653076,
      "loss": 0.5602,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3099935054779053,
      "rewards/margins": 0.1035681962966919,
      "rewards/rejected": -1.4135615825653076,
      "sft_loss": 1.3142729997634888,
      "step": 400
    },
    {
      "epoch": 0.2140826225121258,
      "eval_logits/chosen": 0.252483993768692,
      "eval_logits/rejected": 0.3397251069545746,
      "eval_logps/chosen": -1.3432023525238037,
      "eval_logps/rejected": -1.480107069015503,
      "eval_loss": 0.5592606663703918,
      "eval_rewards/accuracies": 0.5578634738922119,
      "eval_rewards/chosen": -1.3432023525238037,
      "eval_rewards/margins": 0.13690461218357086,
      "eval_rewards/rejected": -1.480107069015503,
      "eval_runtime": 43.9397,
      "eval_samples_per_second": 30.61,
      "eval_sft_loss": 1.370203971862793,
      "eval_steps_per_second": 7.67,
      "step": 400
    },
    {
      "epoch": 0.21675865529352734,
      "grad_norm": 7.034234666380894,
      "learning_rate": 7.219251336898395e-07,
      "logits/chosen": -0.027093514800071716,
      "logits/rejected": 0.06692580878734589,
      "logps/chosen": -1.3181028366088867,
      "logps/rejected": -1.389564871788025,
      "loss": 0.5726,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.3181028366088867,
      "rewards/margins": 0.07146209478378296,
      "rewards/rejected": -1.389564871788025,
      "sft_loss": 1.316383957862854,
      "step": 405
    },
    {
      "epoch": 0.2194346880749289,
      "grad_norm": 4.841587816363229,
      "learning_rate": 7.30837789661319e-07,
      "logits/chosen": 0.003671199083328247,
      "logits/rejected": 0.13374844193458557,
      "logps/chosen": -1.2869586944580078,
      "logps/rejected": -1.382009506225586,
      "loss": 0.5626,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.2869586944580078,
      "rewards/margins": 0.09505093097686768,
      "rewards/rejected": -1.382009506225586,
      "sft_loss": 1.3342974185943604,
      "step": 410
    },
    {
      "epoch": 0.2221107208563305,
      "grad_norm": 4.276205679053103,
      "learning_rate": 7.397504456327985e-07,
      "logits/chosen": -0.01809159852564335,
      "logits/rejected": 0.020652102306485176,
      "logps/chosen": -1.28365957736969,
      "logps/rejected": -1.4585318565368652,
      "loss": 0.5463,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.28365957736969,
      "rewards/margins": 0.17487232387065887,
      "rewards/rejected": -1.4585318565368652,
      "sft_loss": 1.3000603914260864,
      "step": 415
    },
    {
      "epoch": 0.22478675363773207,
      "grad_norm": 5.072655211143472,
      "learning_rate": 7.486631016042781e-07,
      "logits/chosen": -0.03806052356958389,
      "logits/rejected": 0.14919230341911316,
      "logps/chosen": -1.273961067199707,
      "logps/rejected": -1.3729407787322998,
      "loss": 0.5628,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.273961067199707,
      "rewards/margins": 0.09897960722446442,
      "rewards/rejected": -1.3729407787322998,
      "sft_loss": 1.3288816213607788,
      "step": 420
    },
    {
      "epoch": 0.22746278641913364,
      "grad_norm": 4.995296057342031,
      "learning_rate": 7.575757575757575e-07,
      "logits/chosen": -0.06995834410190582,
      "logits/rejected": 0.1246950775384903,
      "logps/chosen": -1.313232660293579,
      "logps/rejected": -1.4796546697616577,
      "loss": 0.5382,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.313232660293579,
      "rewards/margins": 0.16642197966575623,
      "rewards/rejected": -1.4796546697616577,
      "sft_loss": 1.3914697170257568,
      "step": 425
    },
    {
      "epoch": 0.2301388192005352,
      "grad_norm": 5.318848179106577,
      "learning_rate": 7.664884135472371e-07,
      "logits/chosen": -0.09253229945898056,
      "logits/rejected": 0.10250599682331085,
      "logps/chosen": -1.3457051515579224,
      "logps/rejected": -1.4839556217193604,
      "loss": 0.5495,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3457051515579224,
      "rewards/margins": 0.1382504254579544,
      "rewards/rejected": -1.4839556217193604,
      "sft_loss": 1.3906058073043823,
      "step": 430
    },
    {
      "epoch": 0.23281485198193677,
      "grad_norm": 6.2690432389327375,
      "learning_rate": 7.754010695187165e-07,
      "logits/chosen": -0.028032371774315834,
      "logits/rejected": 0.0593259334564209,
      "logps/chosen": -1.2195804119110107,
      "logps/rejected": -1.3592599630355835,
      "loss": 0.5264,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.2195804119110107,
      "rewards/margins": 0.13967978954315186,
      "rewards/rejected": -1.3592599630355835,
      "sft_loss": 1.2889639139175415,
      "step": 435
    },
    {
      "epoch": 0.23549088476333835,
      "grad_norm": 4.38473816919049,
      "learning_rate": 7.84313725490196e-07,
      "logits/chosen": -0.03974657505750656,
      "logits/rejected": 0.04667690023779869,
      "logps/chosen": -1.2953468561172485,
      "logps/rejected": -1.3932148218154907,
      "loss": 0.5548,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.2953468561172485,
      "rewards/margins": 0.09786802530288696,
      "rewards/rejected": -1.3932148218154907,
      "sft_loss": 1.3165804147720337,
      "step": 440
    },
    {
      "epoch": 0.23816691754473993,
      "grad_norm": 5.461131482304206,
      "learning_rate": 7.932263814616755e-07,
      "logits/chosen": -0.06804539263248444,
      "logits/rejected": 0.042102426290512085,
      "logps/chosen": -1.324836254119873,
      "logps/rejected": -1.4605040550231934,
      "loss": 0.57,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.324836254119873,
      "rewards/margins": 0.13566775619983673,
      "rewards/rejected": -1.4605040550231934,
      "sft_loss": 1.3660962581634521,
      "step": 445
    },
    {
      "epoch": 0.2408429503261415,
      "grad_norm": 7.46942560698112,
      "learning_rate": 8.02139037433155e-07,
      "logits/chosen": -0.02956671081483364,
      "logits/rejected": 0.09420625865459442,
      "logps/chosen": -1.3139501810073853,
      "logps/rejected": -1.4573780298233032,
      "loss": 0.5296,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3139501810073853,
      "rewards/margins": 0.14342793822288513,
      "rewards/rejected": -1.4573780298233032,
      "sft_loss": 1.301524043083191,
      "step": 450
    },
    {
      "epoch": 0.24351898310754308,
      "grad_norm": 5.766091566763157,
      "learning_rate": 8.110516934046346e-07,
      "logits/chosen": -0.008473304100334644,
      "logits/rejected": 0.08258920907974243,
      "logps/chosen": -1.2512413263320923,
      "logps/rejected": -1.4463129043579102,
      "loss": 0.5305,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.2512413263320923,
      "rewards/margins": 0.1950714886188507,
      "rewards/rejected": -1.4463129043579102,
      "sft_loss": 1.2772114276885986,
      "step": 455
    },
    {
      "epoch": 0.24619501588894463,
      "grad_norm": 5.822042326452618,
      "learning_rate": 8.19964349376114e-07,
      "logits/chosen": -0.13016703724861145,
      "logits/rejected": -0.006366625428199768,
      "logps/chosen": -1.378727912902832,
      "logps/rejected": -1.4377683401107788,
      "loss": 0.594,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.378727912902832,
      "rewards/margins": 0.059040505439043045,
      "rewards/rejected": -1.4377683401107788,
      "sft_loss": 1.4157121181488037,
      "step": 460
    },
    {
      "epoch": 0.2488710486703462,
      "grad_norm": 6.125179452056327,
      "learning_rate": 8.288770053475936e-07,
      "logits/chosen": 0.12094781547784805,
      "logits/rejected": 0.14069999754428864,
      "logps/chosen": -1.286008596420288,
      "logps/rejected": -1.4609568119049072,
      "loss": 0.5365,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.286008596420288,
      "rewards/margins": 0.1749483048915863,
      "rewards/rejected": -1.4609568119049072,
      "sft_loss": 1.2957916259765625,
      "step": 465
    },
    {
      "epoch": 0.2515470814517478,
      "grad_norm": 6.032620914644976,
      "learning_rate": 8.37789661319073e-07,
      "logits/chosen": 0.14457985758781433,
      "logits/rejected": 0.10018442571163177,
      "logps/chosen": -1.2448914051055908,
      "logps/rejected": -1.4428813457489014,
      "loss": 0.5237,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.2448914051055908,
      "rewards/margins": 0.19798986613750458,
      "rewards/rejected": -1.4428813457489014,
      "sft_loss": 1.2870714664459229,
      "step": 470
    },
    {
      "epoch": 0.25422311423314936,
      "grad_norm": 5.5865279276737665,
      "learning_rate": 8.467023172905525e-07,
      "logits/chosen": -0.07837694883346558,
      "logits/rejected": 0.06283046305179596,
      "logps/chosen": -1.3094193935394287,
      "logps/rejected": -1.5276954174041748,
      "loss": 0.5274,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3094193935394287,
      "rewards/margins": 0.2182762175798416,
      "rewards/rejected": -1.5276954174041748,
      "sft_loss": 1.3427727222442627,
      "step": 475
    },
    {
      "epoch": 0.2568991470145509,
      "grad_norm": 5.435895867877187,
      "learning_rate": 8.55614973262032e-07,
      "logits/chosen": -0.055062223225831985,
      "logits/rejected": 0.14435690641403198,
      "logps/chosen": -1.2810598611831665,
      "logps/rejected": -1.3541405200958252,
      "loss": 0.5499,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.2810598611831665,
      "rewards/margins": 0.07308082282543182,
      "rewards/rejected": -1.3541405200958252,
      "sft_loss": 1.3179230690002441,
      "step": 480
    },
    {
      "epoch": 0.2595751797959525,
      "grad_norm": 7.476411782783797,
      "learning_rate": 8.645276292335115e-07,
      "logits/chosen": -0.0025172666646540165,
      "logits/rejected": 0.03772548958659172,
      "logps/chosen": -1.3743191957473755,
      "logps/rejected": -1.4703030586242676,
      "loss": 0.5768,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3743191957473755,
      "rewards/margins": 0.09598368406295776,
      "rewards/rejected": -1.4703030586242676,
      "sft_loss": 1.3797461986541748,
      "step": 485
    },
    {
      "epoch": 0.26225121257735406,
      "grad_norm": 5.044411300219118,
      "learning_rate": 8.734402852049911e-07,
      "logits/chosen": -0.021833175793290138,
      "logits/rejected": 0.050391364842653275,
      "logps/chosen": -1.3183096647262573,
      "logps/rejected": -1.3980544805526733,
      "loss": 0.5845,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.3183096647262573,
      "rewards/margins": 0.07974480092525482,
      "rewards/rejected": -1.3980544805526733,
      "sft_loss": 1.319129228591919,
      "step": 490
    },
    {
      "epoch": 0.26492724535875567,
      "grad_norm": 6.6379109376317516,
      "learning_rate": 8.823529411764705e-07,
      "logits/chosen": -0.05253653600811958,
      "logits/rejected": -0.027758020907640457,
      "logps/chosen": -1.3289823532104492,
      "logps/rejected": -1.44633948802948,
      "loss": 0.5614,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.3289823532104492,
      "rewards/margins": 0.11735711246728897,
      "rewards/rejected": -1.44633948802948,
      "sft_loss": 1.3961108922958374,
      "step": 495
    },
    {
      "epoch": 0.2676032781401572,
      "grad_norm": 5.349956993841474,
      "learning_rate": 8.912655971479501e-07,
      "logits/chosen": -0.05144606903195381,
      "logits/rejected": 0.048278652131557465,
      "logps/chosen": -1.235568642616272,
      "logps/rejected": -1.3974435329437256,
      "loss": 0.5408,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.235568642616272,
      "rewards/margins": 0.16187506914138794,
      "rewards/rejected": -1.3974435329437256,
      "sft_loss": 1.2869036197662354,
      "step": 500
    },
    {
      "epoch": 0.27027931092155877,
      "grad_norm": 8.559922082272234,
      "learning_rate": 9.001782531194295e-07,
      "logits/chosen": -0.05748797580599785,
      "logits/rejected": 0.08330624550580978,
      "logps/chosen": -1.3548730611801147,
      "logps/rejected": -1.4059706926345825,
      "loss": 0.5894,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.3548730611801147,
      "rewards/margins": 0.05109750106930733,
      "rewards/rejected": -1.4059706926345825,
      "sft_loss": 1.38583242893219,
      "step": 505
    },
    {
      "epoch": 0.2729553437029604,
      "grad_norm": 5.775228165472396,
      "learning_rate": 9.09090909090909e-07,
      "logits/chosen": 0.09095264971256256,
      "logits/rejected": 0.15189726650714874,
      "logps/chosen": -1.3080486059188843,
      "logps/rejected": -1.485636591911316,
      "loss": 0.5364,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3080486059188843,
      "rewards/margins": 0.1775878518819809,
      "rewards/rejected": -1.485636591911316,
      "sft_loss": 1.2965028285980225,
      "step": 510
    },
    {
      "epoch": 0.2756313764843619,
      "grad_norm": 4.144221629537718,
      "learning_rate": 9.180035650623885e-07,
      "logits/chosen": 0.05798070505261421,
      "logits/rejected": 0.15644438564777374,
      "logps/chosen": -1.249235987663269,
      "logps/rejected": -1.4133098125457764,
      "loss": 0.5357,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.249235987663269,
      "rewards/margins": 0.16407367587089539,
      "rewards/rejected": -1.4133098125457764,
      "sft_loss": 1.2908960580825806,
      "step": 515
    },
    {
      "epoch": 0.27830740926576353,
      "grad_norm": 4.279287128656563,
      "learning_rate": 9.26916221033868e-07,
      "logits/chosen": -0.0792783722281456,
      "logits/rejected": 0.0598064549267292,
      "logps/chosen": -1.2978101968765259,
      "logps/rejected": -1.4097940921783447,
      "loss": 0.5585,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.2978101968765259,
      "rewards/margins": 0.1119840145111084,
      "rewards/rejected": -1.4097940921783447,
      "sft_loss": 1.4094016551971436,
      "step": 520
    },
    {
      "epoch": 0.2809834420471651,
      "grad_norm": 10.53445430162785,
      "learning_rate": 9.358288770053476e-07,
      "logits/chosen": 0.1093759760260582,
      "logits/rejected": 0.18354789912700653,
      "logps/chosen": -1.275532841682434,
      "logps/rejected": -1.4687607288360596,
      "loss": 0.5423,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.275532841682434,
      "rewards/margins": 0.193228080868721,
      "rewards/rejected": -1.4687607288360596,
      "sft_loss": 1.373024344444275,
      "step": 525
    },
    {
      "epoch": 0.2836594748285666,
      "grad_norm": 4.390986869790019,
      "learning_rate": 9.44741532976827e-07,
      "logits/chosen": 0.08234995603561401,
      "logits/rejected": 0.1688038408756256,
      "logps/chosen": -1.2554372549057007,
      "logps/rejected": -1.3672778606414795,
      "loss": 0.5493,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.2554372549057007,
      "rewards/margins": 0.11184068024158478,
      "rewards/rejected": -1.3672778606414795,
      "sft_loss": 1.230678915977478,
      "step": 530
    },
    {
      "epoch": 0.28633550760996823,
      "grad_norm": 4.469595540150405,
      "learning_rate": 9.536541889483066e-07,
      "logits/chosen": -0.08351071178913116,
      "logits/rejected": 0.18095479905605316,
      "logps/chosen": -1.2669923305511475,
      "logps/rejected": -1.3454608917236328,
      "loss": 0.553,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.2669923305511475,
      "rewards/margins": 0.07846838980913162,
      "rewards/rejected": -1.3454608917236328,
      "sft_loss": 1.253920555114746,
      "step": 535
    },
    {
      "epoch": 0.2890115403913698,
      "grad_norm": 4.434656541244071,
      "learning_rate": 9.62566844919786e-07,
      "logits/chosen": 0.04407946765422821,
      "logits/rejected": 0.1207633838057518,
      "logps/chosen": -1.396340250968933,
      "logps/rejected": -1.4665273427963257,
      "loss": 0.5854,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.396340250968933,
      "rewards/margins": 0.07018700987100601,
      "rewards/rejected": -1.4665273427963257,
      "sft_loss": 1.4373433589935303,
      "step": 540
    },
    {
      "epoch": 0.2916875731727714,
      "grad_norm": 4.136588625276095,
      "learning_rate": 9.714795008912655e-07,
      "logits/chosen": -0.07348589599132538,
      "logits/rejected": 0.1302313506603241,
      "logps/chosen": -1.299394965171814,
      "logps/rejected": -1.4257206916809082,
      "loss": 0.5352,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.299394965171814,
      "rewards/margins": 0.12632553279399872,
      "rewards/rejected": -1.4257206916809082,
      "sft_loss": 1.324333906173706,
      "step": 545
    },
    {
      "epoch": 0.29436360595417294,
      "grad_norm": 4.352118373128978,
      "learning_rate": 9.80392156862745e-07,
      "logits/chosen": 0.07032302767038345,
      "logits/rejected": 0.14239667356014252,
      "logps/chosen": -1.2955199480056763,
      "logps/rejected": -1.4420729875564575,
      "loss": 0.5384,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.2955199480056763,
      "rewards/margins": 0.1465531587600708,
      "rewards/rejected": -1.4420729875564575,
      "sft_loss": 1.2931840419769287,
      "step": 550
    },
    {
      "epoch": 0.2970396387355745,
      "grad_norm": 6.622239099763008,
      "learning_rate": 9.893048128342244e-07,
      "logits/chosen": -0.059581220149993896,
      "logits/rejected": 0.06685901433229446,
      "logps/chosen": -1.3705341815948486,
      "logps/rejected": -1.4440644979476929,
      "loss": 0.595,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3705341815948486,
      "rewards/margins": 0.0735301673412323,
      "rewards/rejected": -1.4440644979476929,
      "sft_loss": 1.3887046575546265,
      "step": 555
    },
    {
      "epoch": 0.2997156715169761,
      "grad_norm": 5.616977527800097,
      "learning_rate": 9.98217468805704e-07,
      "logits/chosen": 0.03694029897451401,
      "logits/rejected": 0.05506284162402153,
      "logps/chosen": -1.2274644374847412,
      "logps/rejected": -1.3752453327178955,
      "loss": 0.5245,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.2274644374847412,
      "rewards/margins": 0.14778101444244385,
      "rewards/rejected": -1.3752453327178955,
      "sft_loss": 1.3644822835922241,
      "step": 560
    },
    {
      "epoch": 0.30239170429837764,
      "grad_norm": 3.896688294517457,
      "learning_rate": 9.999984476788462e-07,
      "logits/chosen": 0.033395491540431976,
      "logits/rejected": 0.09102457761764526,
      "logps/chosen": -1.3448399305343628,
      "logps/rejected": -1.4782801866531372,
      "loss": 0.5593,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.3448399305343628,
      "rewards/margins": 0.13344022631645203,
      "rewards/rejected": -1.4782801866531372,
      "sft_loss": 1.3826723098754883,
      "step": 565
    },
    {
      "epoch": 0.30506773707977924,
      "grad_norm": 6.468949859109518,
      "learning_rate": 9.999921413906797e-07,
      "logits/chosen": -0.03206388279795647,
      "logits/rejected": 0.1903473287820816,
      "logps/chosen": -1.3156042098999023,
      "logps/rejected": -1.4189167022705078,
      "loss": 0.5637,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.3156042098999023,
      "rewards/margins": 0.10331261157989502,
      "rewards/rejected": -1.4189167022705078,
      "sft_loss": 1.4111707210540771,
      "step": 570
    },
    {
      "epoch": 0.3077437698611808,
      "grad_norm": 4.715156501261207,
      "learning_rate": 9.999809841765644e-07,
      "logits/chosen": -0.016568869352340698,
      "logits/rejected": 0.04429225996136665,
      "logps/chosen": -1.2501987218856812,
      "logps/rejected": -1.3790241479873657,
      "loss": 0.5316,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.2501987218856812,
      "rewards/margins": 0.128825381398201,
      "rewards/rejected": -1.3790241479873657,
      "sft_loss": 1.2782385349273682,
      "step": 575
    },
    {
      "epoch": 0.3104198026425824,
      "grad_norm": 5.367916524502688,
      "learning_rate": 9.999649761447477e-07,
      "logits/chosen": -0.01917845942080021,
      "logits/rejected": 0.1418890506029129,
      "logps/chosen": -1.2578179836273193,
      "logps/rejected": -1.440937876701355,
      "loss": 0.5176,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.2578179836273193,
      "rewards/margins": 0.18311990797519684,
      "rewards/rejected": -1.440937876701355,
      "sft_loss": 1.2896482944488525,
      "step": 580
    },
    {
      "epoch": 0.31309583542398395,
      "grad_norm": 5.914173986510269,
      "learning_rate": 9.999441174505398e-07,
      "logits/chosen": -0.07606562972068787,
      "logits/rejected": 0.027116943150758743,
      "logps/chosen": -1.3965537548065186,
      "logps/rejected": -1.4706159830093384,
      "loss": 0.5945,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3965537548065186,
      "rewards/margins": 0.07406215369701385,
      "rewards/rejected": -1.4706159830093384,
      "sft_loss": 1.4381605386734009,
      "step": 585
    },
    {
      "epoch": 0.3157718682053855,
      "grad_norm": 6.775658620216753,
      "learning_rate": 9.999184082963116e-07,
      "logits/chosen": -0.04157830402255058,
      "logits/rejected": 0.08735966682434082,
      "logps/chosen": -1.3702585697174072,
      "logps/rejected": -1.4163644313812256,
      "loss": 0.5928,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.3702585697174072,
      "rewards/margins": 0.0461057610809803,
      "rewards/rejected": -1.4163644313812256,
      "sft_loss": 1.413253664970398,
      "step": 590
    },
    {
      "epoch": 0.3184479009867871,
      "grad_norm": 5.304206518820304,
      "learning_rate": 9.998878489314937e-07,
      "logits/chosen": 0.021392468363046646,
      "logits/rejected": 0.14886918663978577,
      "logps/chosen": -1.3005200624465942,
      "logps/rejected": -1.3912214040756226,
      "loss": 0.5591,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3005200624465942,
      "rewards/margins": 0.09070131927728653,
      "rewards/rejected": -1.3912214040756226,
      "sft_loss": 1.318687915802002,
      "step": 595
    },
    {
      "epoch": 0.32112393376818865,
      "grad_norm": 4.73947769164422,
      "learning_rate": 9.99852439652573e-07,
      "logits/chosen": -0.04575077071785927,
      "logits/rejected": 0.10018666833639145,
      "logps/chosen": -1.2980984449386597,
      "logps/rejected": -1.3406703472137451,
      "loss": 0.5772,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.2980984449386597,
      "rewards/margins": 0.0425720252096653,
      "rewards/rejected": -1.3406703472137451,
      "sft_loss": 1.3395038843154907,
      "step": 600
    },
    {
      "epoch": 0.32379996654959026,
      "grad_norm": 5.043264487838186,
      "learning_rate": 9.998121808030904e-07,
      "logits/chosen": -0.08167050778865814,
      "logits/rejected": 0.004694207105785608,
      "logps/chosen": -1.3602782487869263,
      "logps/rejected": -1.532902479171753,
      "loss": 0.5545,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3602782487869263,
      "rewards/margins": 0.1726243793964386,
      "rewards/rejected": -1.532902479171753,
      "sft_loss": 1.398123025894165,
      "step": 605
    },
    {
      "epoch": 0.3264759993309918,
      "grad_norm": 13.463144768403025,
      "learning_rate": 9.997670727736379e-07,
      "logits/chosen": 0.03331831097602844,
      "logits/rejected": 0.18258699774742126,
      "logps/chosen": -1.3355567455291748,
      "logps/rejected": -1.4422796964645386,
      "loss": 0.5664,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3355567455291748,
      "rewards/margins": 0.10672305524349213,
      "rewards/rejected": -1.4422796964645386,
      "sft_loss": 1.363588571548462,
      "step": 610
    },
    {
      "epoch": 0.32915203211239336,
      "grad_norm": 4.328631685243004,
      "learning_rate": 9.99717116001853e-07,
      "logits/chosen": -0.06358928978443146,
      "logits/rejected": 0.042626697570085526,
      "logps/chosen": -1.3286917209625244,
      "logps/rejected": -1.504015564918518,
      "loss": 0.5522,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3286917209625244,
      "rewards/margins": 0.17532379925251007,
      "rewards/rejected": -1.504015564918518,
      "sft_loss": 1.3659374713897705,
      "step": 615
    },
    {
      "epoch": 0.33182806489379496,
      "grad_norm": 4.821431945232963,
      "learning_rate": 9.996623109724173e-07,
      "logits/chosen": 0.039220355451107025,
      "logits/rejected": 0.10534069687128067,
      "logps/chosen": -1.4000418186187744,
      "logps/rejected": -1.5270695686340332,
      "loss": 0.5804,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.4000418186187744,
      "rewards/margins": 0.12702779471874237,
      "rewards/rejected": -1.5270695686340332,
      "sft_loss": 1.401197075843811,
      "step": 620
    },
    {
      "epoch": 0.3345040976751965,
      "grad_norm": 6.148785201472726,
      "learning_rate": 9.996026582170488e-07,
      "logits/chosen": 0.07166017591953278,
      "logits/rejected": 0.1898956000804901,
      "logps/chosen": -1.3064854145050049,
      "logps/rejected": -1.4716060161590576,
      "loss": 0.5241,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3064854145050049,
      "rewards/margins": 0.1651206910610199,
      "rewards/rejected": -1.4716060161590576,
      "sft_loss": 1.3478180170059204,
      "step": 625
    },
    {
      "epoch": 0.3371801304565981,
      "grad_norm": 6.500178864042233,
      "learning_rate": 9.995381583144996e-07,
      "logits/chosen": -0.015255662612617016,
      "logits/rejected": 0.0960661917924881,
      "logps/chosen": -1.3437426090240479,
      "logps/rejected": -1.5181114673614502,
      "loss": 0.5323,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3437426090240479,
      "rewards/margins": 0.1743689477443695,
      "rewards/rejected": -1.5181114673614502,
      "sft_loss": 1.3270080089569092,
      "step": 630
    },
    {
      "epoch": 0.33985616323799966,
      "grad_norm": 4.284856290724194,
      "learning_rate": 9.994688118905471e-07,
      "logits/chosen": -0.019777098670601845,
      "logits/rejected": 0.2161232978105545,
      "logps/chosen": -1.4191901683807373,
      "logps/rejected": -1.499983310699463,
      "loss": 0.5951,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.4191901683807373,
      "rewards/margins": 0.08079305291175842,
      "rewards/rejected": -1.499983310699463,
      "sft_loss": 1.4321720600128174,
      "step": 635
    },
    {
      "epoch": 0.3425321960194012,
      "grad_norm": 12.656078393455692,
      "learning_rate": 9.993946196179912e-07,
      "logits/chosen": -0.11304919421672821,
      "logits/rejected": 0.08124849945306778,
      "logps/chosen": -1.3428949117660522,
      "logps/rejected": -1.478323221206665,
      "loss": 0.5509,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3428949117660522,
      "rewards/margins": 0.13542838394641876,
      "rewards/rejected": -1.478323221206665,
      "sft_loss": 1.4283742904663086,
      "step": 640
    },
    {
      "epoch": 0.3452082288008028,
      "grad_norm": 5.579514201615282,
      "learning_rate": 9.993155822166455e-07,
      "logits/chosen": -0.09503932297229767,
      "logits/rejected": -0.008171332068741322,
      "logps/chosen": -1.2692936658859253,
      "logps/rejected": -1.471149206161499,
      "loss": 0.5065,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.2692936658859253,
      "rewards/margins": 0.2018553465604782,
      "rewards/rejected": -1.471149206161499,
      "sft_loss": 1.2678296566009521,
      "step": 645
    },
    {
      "epoch": 0.34788426158220437,
      "grad_norm": 7.144522751822136,
      "learning_rate": 9.992317004533313e-07,
      "logits/chosen": -0.015544240362942219,
      "logits/rejected": 0.12737944722175598,
      "logps/chosen": -1.4104164838790894,
      "logps/rejected": -1.5921103954315186,
      "loss": 0.5565,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.4104164838790894,
      "rewards/margins": 0.18169382214546204,
      "rewards/rejected": -1.5921103954315186,
      "sft_loss": 1.428065299987793,
      "step": 650
    },
    {
      "epoch": 0.350560294363606,
      "grad_norm": 6.373414994710634,
      "learning_rate": 9.991429751418696e-07,
      "logits/chosen": 0.033987972885370255,
      "logits/rejected": 0.04610954597592354,
      "logps/chosen": -1.3487951755523682,
      "logps/rejected": -1.5615766048431396,
      "loss": 0.5584,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.3487951755523682,
      "rewards/margins": 0.21278128027915955,
      "rewards/rejected": -1.5615766048431396,
      "sft_loss": 1.366637945175171,
      "step": 655
    },
    {
      "epoch": 0.3532363271450075,
      "grad_norm": 5.19965274289491,
      "learning_rate": 9.99049407143074e-07,
      "logits/chosen": 0.014334109611809254,
      "logits/rejected": 0.14489109814167023,
      "logps/chosen": -1.3145679235458374,
      "logps/rejected": -1.390172004699707,
      "loss": 0.5607,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.3145679235458374,
      "rewards/margins": 0.07560417801141739,
      "rewards/rejected": -1.390172004699707,
      "sft_loss": 1.3641953468322754,
      "step": 660
    },
    {
      "epoch": 0.35591235992640907,
      "grad_norm": 6.443123859185252,
      "learning_rate": 9.989509973647416e-07,
      "logits/chosen": 0.007931932806968689,
      "logits/rejected": 0.1458900421857834,
      "logps/chosen": -1.2717044353485107,
      "logps/rejected": -1.4361180067062378,
      "loss": 0.5326,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.2717044353485107,
      "rewards/margins": 0.1644134223461151,
      "rewards/rejected": -1.4361180067062378,
      "sft_loss": 1.3242876529693604,
      "step": 665
    },
    {
      "epoch": 0.3585883927078107,
      "grad_norm": 4.619542183407082,
      "learning_rate": 9.988477467616445e-07,
      "logits/chosen": -0.017918167635798454,
      "logits/rejected": 0.1990184336900711,
      "logps/chosen": -1.3112833499908447,
      "logps/rejected": -1.3996978998184204,
      "loss": 0.5527,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -1.3112833499908447,
      "rewards/margins": 0.08841459453105927,
      "rewards/rejected": -1.3996978998184204,
      "sft_loss": 1.4118869304656982,
      "step": 670
    },
    {
      "epoch": 0.3612644254892122,
      "grad_norm": 5.341412538195278,
      "learning_rate": 9.987396563355205e-07,
      "logits/chosen": -0.03343156352639198,
      "logits/rejected": 0.04960675165057182,
      "logps/chosen": -1.3060057163238525,
      "logps/rejected": -1.5539497137069702,
      "loss": 0.513,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.3060057163238525,
      "rewards/margins": 0.24794375896453857,
      "rewards/rejected": -1.5539497137069702,
      "sft_loss": 1.3633852005004883,
      "step": 675
    },
    {
      "epoch": 0.36394045827061383,
      "grad_norm": 5.013306968245386,
      "learning_rate": 9.986267271350631e-07,
      "logits/chosen": 0.07540982961654663,
      "logits/rejected": 0.23975494503974915,
      "logps/chosen": -1.357664704322815,
      "logps/rejected": -1.4599559307098389,
      "loss": 0.6013,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.357664704322815,
      "rewards/margins": 0.10229126363992691,
      "rewards/rejected": -1.4599559307098389,
      "sft_loss": 1.3449851274490356,
      "step": 680
    },
    {
      "epoch": 0.3666164910520154,
      "grad_norm": 7.835812606614032,
      "learning_rate": 9.985089602559123e-07,
      "logits/chosen": 0.02607138082385063,
      "logits/rejected": 0.1844528168439865,
      "logps/chosen": -1.329110860824585,
      "logps/rejected": -1.4306621551513672,
      "loss": 0.5641,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.329110860824585,
      "rewards/margins": 0.10155129432678223,
      "rewards/rejected": -1.4306621551513672,
      "sft_loss": 1.3462426662445068,
      "step": 685
    },
    {
      "epoch": 0.369292523833417,
      "grad_norm": 6.033085338974475,
      "learning_rate": 9.983863568406428e-07,
      "logits/chosen": 0.059251852333545685,
      "logits/rejected": 0.09896888583898544,
      "logps/chosen": -1.3339654207229614,
      "logps/rejected": -1.4830416440963745,
      "loss": 0.5704,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.3339654207229614,
      "rewards/margins": 0.14907608926296234,
      "rewards/rejected": -1.4830416440963745,
      "sft_loss": 1.3890200853347778,
      "step": 690
    },
    {
      "epoch": 0.37196855661481854,
      "grad_norm": 4.424859213314543,
      "learning_rate": 9.982589180787532e-07,
      "logits/chosen": 0.008425706997513771,
      "logits/rejected": 0.10140939801931381,
      "logps/chosen": -1.2267965078353882,
      "logps/rejected": -1.4236056804656982,
      "loss": 0.5129,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.2267965078353882,
      "rewards/margins": 0.1968090832233429,
      "rewards/rejected": -1.4236056804656982,
      "sft_loss": 1.2810438871383667,
      "step": 695
    },
    {
      "epoch": 0.3746445893962201,
      "grad_norm": 6.692631922724751,
      "learning_rate": 9.981266452066553e-07,
      "logits/chosen": -0.09820245206356049,
      "logits/rejected": 0.047392718493938446,
      "logps/chosen": -1.4009666442871094,
      "logps/rejected": -1.5001310110092163,
      "loss": 0.5768,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.4009666442871094,
      "rewards/margins": 0.09916438907384872,
      "rewards/rejected": -1.5001310110092163,
      "sft_loss": 1.385056972503662,
      "step": 700
    },
    {
      "epoch": 0.3773206221776217,
      "grad_norm": 4.951811140853431,
      "learning_rate": 9.979895395076608e-07,
      "logits/chosen": -0.07248760014772415,
      "logits/rejected": 0.10724302381277084,
      "logps/chosen": -1.336701512336731,
      "logps/rejected": -1.547623872756958,
      "loss": 0.5249,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.336701512336731,
      "rewards/margins": 0.21092236042022705,
      "rewards/rejected": -1.547623872756958,
      "sft_loss": 1.3786590099334717,
      "step": 705
    },
    {
      "epoch": 0.37999665495902324,
      "grad_norm": 4.818916350234281,
      "learning_rate": 9.9784760231197e-07,
      "logits/chosen": 0.05171216279268265,
      "logits/rejected": 0.14952951669692993,
      "logps/chosen": -1.2990772724151611,
      "logps/rejected": -1.4701385498046875,
      "loss": 0.5232,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.2990772724151611,
      "rewards/margins": 0.1710611879825592,
      "rewards/rejected": -1.4701385498046875,
      "sft_loss": 1.3045198917388916,
      "step": 710
    },
    {
      "epoch": 0.38267268774042484,
      "grad_norm": 7.471676922467004,
      "learning_rate": 9.97700834996658e-07,
      "logits/chosen": -0.012614324688911438,
      "logits/rejected": 0.1592131406068802,
      "logps/chosen": -1.371006965637207,
      "logps/rejected": -1.528407335281372,
      "loss": 0.5366,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.371006965637207,
      "rewards/margins": 0.1574004739522934,
      "rewards/rejected": -1.528407335281372,
      "sft_loss": 1.3581221103668213,
      "step": 715
    },
    {
      "epoch": 0.3853487205218264,
      "grad_norm": 5.855309953913271,
      "learning_rate": 9.97549238985662e-07,
      "logits/chosen": 0.05896524339914322,
      "logits/rejected": 0.2562909722328186,
      "logps/chosen": -1.4231863021850586,
      "logps/rejected": -1.5625927448272705,
      "loss": 0.5803,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4231863021850586,
      "rewards/margins": 0.13940641283988953,
      "rewards/rejected": -1.5625927448272705,
      "sft_loss": 1.4524192810058594,
      "step": 720
    },
    {
      "epoch": 0.38802475330322794,
      "grad_norm": 5.623027773331276,
      "learning_rate": 9.973928157497674e-07,
      "logits/chosen": -0.09292992949485779,
      "logits/rejected": 0.04666275531053543,
      "logps/chosen": -1.2506234645843506,
      "logps/rejected": -1.5481257438659668,
      "loss": 0.4816,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.2506234645843506,
      "rewards/margins": 0.2975021004676819,
      "rewards/rejected": -1.5481257438659668,
      "sft_loss": 1.323334813117981,
      "step": 725
    },
    {
      "epoch": 0.39070078608462955,
      "grad_norm": 5.552946198660787,
      "learning_rate": 9.972315668065927e-07,
      "logits/chosen": -0.13729050755500793,
      "logits/rejected": 0.02584907039999962,
      "logps/chosen": -1.364895224571228,
      "logps/rejected": -1.5133196115493774,
      "loss": 0.5632,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.364895224571228,
      "rewards/margins": 0.14842435717582703,
      "rewards/rejected": -1.5133196115493774,
      "sft_loss": 1.3803001642227173,
      "step": 730
    },
    {
      "epoch": 0.3933768188660311,
      "grad_norm": 5.09955935784875,
      "learning_rate": 9.97065493720576e-07,
      "logits/chosen": -0.10641257464885712,
      "logits/rejected": -0.002804142190143466,
      "logps/chosen": -1.3838285207748413,
      "logps/rejected": -1.5112088918685913,
      "loss": 0.5461,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3838285207748413,
      "rewards/margins": 0.12738032639026642,
      "rewards/rejected": -1.5112088918685913,
      "sft_loss": 1.4276145696640015,
      "step": 735
    },
    {
      "epoch": 0.3960528516474327,
      "grad_norm": 8.08936145133186,
      "learning_rate": 9.968945981029594e-07,
      "logits/chosen": -0.0654737576842308,
      "logits/rejected": 0.11496500670909882,
      "logps/chosen": -1.4559099674224854,
      "logps/rejected": -1.5509315729141235,
      "loss": 0.5959,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.4559099674224854,
      "rewards/margins": 0.09502150863409042,
      "rewards/rejected": -1.5509315729141235,
      "sft_loss": 1.4707705974578857,
      "step": 740
    },
    {
      "epoch": 0.39872888442883425,
      "grad_norm": 4.860947110964878,
      "learning_rate": 9.967188816117726e-07,
      "logits/chosen": 0.03588312119245529,
      "logits/rejected": 0.11505071818828583,
      "logps/chosen": -1.4208877086639404,
      "logps/rejected": -1.675723671913147,
      "loss": 0.569,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4208877086639404,
      "rewards/margins": 0.2548360228538513,
      "rewards/rejected": -1.675723671913147,
      "sft_loss": 1.4088971614837646,
      "step": 745
    },
    {
      "epoch": 0.4014049172102358,
      "grad_norm": 5.085383768228615,
      "learning_rate": 9.965383459518179e-07,
      "logits/chosen": -0.0216512531042099,
      "logits/rejected": 0.14498159289360046,
      "logps/chosen": -1.360144853591919,
      "logps/rejected": -1.5747146606445312,
      "loss": 0.5322,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.360144853591919,
      "rewards/margins": 0.2145698070526123,
      "rewards/rejected": -1.5747146606445312,
      "sft_loss": 1.3781875371932983,
      "step": 750
    },
    {
      "epoch": 0.4040809499916374,
      "grad_norm": 5.0297073385361895,
      "learning_rate": 9.963529928746533e-07,
      "logits/chosen": 0.009959183633327484,
      "logits/rejected": 0.14851178228855133,
      "logps/chosen": -1.3677583932876587,
      "logps/rejected": -1.519020915031433,
      "loss": 0.568,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.3677583932876587,
      "rewards/margins": 0.15126249194145203,
      "rewards/rejected": -1.519020915031433,
      "sft_loss": 1.408804178237915,
      "step": 755
    },
    {
      "epoch": 0.40675698277303896,
      "grad_norm": 4.209994488263506,
      "learning_rate": 9.961628241785746e-07,
      "logits/chosen": -0.07662223279476166,
      "logits/rejected": -0.00010142624523723498,
      "logps/chosen": -1.3832899332046509,
      "logps/rejected": -1.592454195022583,
      "loss": 0.5372,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3832899332046509,
      "rewards/margins": 0.2091643512248993,
      "rewards/rejected": -1.592454195022583,
      "sft_loss": 1.423018217086792,
      "step": 760
    },
    {
      "epoch": 0.40943301555444056,
      "grad_norm": 5.371155698509368,
      "learning_rate": 9.959678417085998e-07,
      "logits/chosen": -0.03791964054107666,
      "logits/rejected": 0.05540591478347778,
      "logps/chosen": -1.3681023120880127,
      "logps/rejected": -1.5076701641082764,
      "loss": 0.5507,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3681023120880127,
      "rewards/margins": 0.13956794142723083,
      "rewards/rejected": -1.5076701641082764,
      "sft_loss": 1.3541196584701538,
      "step": 765
    },
    {
      "epoch": 0.4121090483358421,
      "grad_norm": 6.2165730754532476,
      "learning_rate": 9.957680473564493e-07,
      "logits/chosen": 0.033802516758441925,
      "logits/rejected": 0.1595228612422943,
      "logps/chosen": -1.3133952617645264,
      "logps/rejected": -1.6211875677108765,
      "loss": 0.5169,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3133952617645264,
      "rewards/margins": 0.30779242515563965,
      "rewards/rejected": -1.6211875677108765,
      "sft_loss": 1.3352806568145752,
      "step": 770
    },
    {
      "epoch": 0.41478508111724366,
      "grad_norm": 5.954812711931833,
      "learning_rate": 9.95563443060529e-07,
      "logits/chosen": -0.09634337574243546,
      "logits/rejected": 0.07502253353595734,
      "logps/chosen": -1.38030207157135,
      "logps/rejected": -1.5933905839920044,
      "loss": 0.55,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.38030207157135,
      "rewards/margins": 0.2130887806415558,
      "rewards/rejected": -1.5933905839920044,
      "sft_loss": 1.3440942764282227,
      "step": 775
    },
    {
      "epoch": 0.41746111389864526,
      "grad_norm": 4.092097929588904,
      "learning_rate": 9.95354030805911e-07,
      "logits/chosen": -0.13427576422691345,
      "logits/rejected": 0.01720806211233139,
      "logps/chosen": -1.3094775676727295,
      "logps/rejected": -1.5330116748809814,
      "loss": 0.5181,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3094775676727295,
      "rewards/margins": 0.22353395819664001,
      "rewards/rejected": -1.5330116748809814,
      "sft_loss": 1.3614782094955444,
      "step": 780
    },
    {
      "epoch": 0.4201371466800468,
      "grad_norm": 4.773573559503689,
      "learning_rate": 9.951398126243133e-07,
      "logits/chosen": 0.010130161419510841,
      "logits/rejected": 0.13926295936107635,
      "logps/chosen": -1.2965346574783325,
      "logps/rejected": -1.5689843893051147,
      "loss": 0.5111,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.2965346574783325,
      "rewards/margins": 0.2724498510360718,
      "rewards/rejected": -1.5689843893051147,
      "sft_loss": 1.3222274780273438,
      "step": 785
    },
    {
      "epoch": 0.4228131794614484,
      "grad_norm": 5.362179923085685,
      "learning_rate": 9.94920790594082e-07,
      "logits/chosen": -0.06429972499608994,
      "logits/rejected": 0.06651268899440765,
      "logps/chosen": -1.351266622543335,
      "logps/rejected": -1.5239372253417969,
      "loss": 0.5427,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.351266622543335,
      "rewards/margins": 0.1726706176996231,
      "rewards/rejected": -1.5239372253417969,
      "sft_loss": 1.3542873859405518,
      "step": 790
    },
    {
      "epoch": 0.42548921224284997,
      "grad_norm": 6.912654885108542,
      "learning_rate": 9.946969668401696e-07,
      "logits/chosen": -0.053594689816236496,
      "logits/rejected": 0.14611445367336273,
      "logps/chosen": -1.334825873374939,
      "logps/rejected": -1.5915372371673584,
      "loss": 0.5339,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.334825873374939,
      "rewards/margins": 0.2567111849784851,
      "rewards/rejected": -1.5915372371673584,
      "sft_loss": 1.3713914155960083,
      "step": 795
    },
    {
      "epoch": 0.4281652450242516,
      "grad_norm": 5.134436625838364,
      "learning_rate": 9.944683435341155e-07,
      "logits/chosen": -0.02290893718600273,
      "logits/rejected": 0.0638694167137146,
      "logps/chosen": -1.3296618461608887,
      "logps/rejected": -1.4925134181976318,
      "loss": 0.5448,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3296618461608887,
      "rewards/margins": 0.16285140812397003,
      "rewards/rejected": -1.4925134181976318,
      "sft_loss": 1.3362598419189453,
      "step": 800
    },
    {
      "epoch": 0.4281652450242516,
      "eval_logits/chosen": 0.272663950920105,
      "eval_logits/rejected": 0.3617731034755707,
      "eval_logps/chosen": -1.3784986734390259,
      "eval_logps/rejected": -1.6209443807601929,
      "eval_loss": 0.5318129062652588,
      "eval_rewards/accuracies": 0.5882789492607117,
      "eval_rewards/chosen": -1.3784986734390259,
      "eval_rewards/margins": 0.24244572222232819,
      "eval_rewards/rejected": -1.6209443807601929,
      "eval_runtime": 43.3554,
      "eval_samples_per_second": 31.023,
      "eval_sft_loss": 1.4044005870819092,
      "eval_steps_per_second": 7.773,
      "step": 800
    },
    {
      "epoch": 0.4308412778056531,
      "grad_norm": 6.714069611575347,
      "learning_rate": 9.942349228940236e-07,
      "logits/chosen": -0.09477131068706512,
      "logits/rejected": 0.06664792448282242,
      "logps/chosen": -1.379688024520874,
      "logps/rejected": -1.6606252193450928,
      "loss": 0.5285,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.379688024520874,
      "rewards/margins": 0.2809372842311859,
      "rewards/rejected": -1.6606252193450928,
      "sft_loss": 1.4098918437957764,
      "step": 805
    },
    {
      "epoch": 0.43351731058705467,
      "grad_norm": 6.023653869387464,
      "learning_rate": 9.939967071845424e-07,
      "logits/chosen": 0.019546739757061005,
      "logits/rejected": 0.0949871689081192,
      "logps/chosen": -1.3077045679092407,
      "logps/rejected": -1.5455154180526733,
      "loss": 0.5199,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.3077045679092407,
      "rewards/margins": 0.23781093955039978,
      "rewards/rejected": -1.5455154180526733,
      "sft_loss": 1.3323251008987427,
      "step": 810
    },
    {
      "epoch": 0.4361933433684563,
      "grad_norm": 7.590492601575631,
      "learning_rate": 9.937536987168413e-07,
      "logits/chosen": 0.01001892052590847,
      "logits/rejected": 0.1464158594608307,
      "logps/chosen": -1.3059206008911133,
      "logps/rejected": -1.6408694982528687,
      "loss": 0.4988,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3059206008911133,
      "rewards/margins": 0.3349488377571106,
      "rewards/rejected": -1.6408694982528687,
      "sft_loss": 1.3800963163375854,
      "step": 815
    },
    {
      "epoch": 0.4388693761498578,
      "grad_norm": 6.846753860809149,
      "learning_rate": 9.935058998485896e-07,
      "logits/chosen": 0.02080114185810089,
      "logits/rejected": 0.07514943182468414,
      "logps/chosen": -1.390001654624939,
      "logps/rejected": -1.6592248678207397,
      "loss": 0.5336,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.390001654624939,
      "rewards/margins": 0.26922309398651123,
      "rewards/rejected": -1.6592248678207397,
      "sft_loss": 1.4036567211151123,
      "step": 820
    },
    {
      "epoch": 0.44154540893125943,
      "grad_norm": 13.520316774007492,
      "learning_rate": 9.932533129839333e-07,
      "logits/chosen": -0.04507501795887947,
      "logits/rejected": 0.08645032346248627,
      "logps/chosen": -1.315307855606079,
      "logps/rejected": -1.5431115627288818,
      "loss": 0.5369,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.315307855606079,
      "rewards/margins": 0.22780366241931915,
      "rewards/rejected": -1.5431115627288818,
      "sft_loss": 1.4078117609024048,
      "step": 825
    },
    {
      "epoch": 0.444221441712661,
      "grad_norm": 6.587293554273986,
      "learning_rate": 9.929959405734711e-07,
      "logits/chosen": 0.06493745744228363,
      "logits/rejected": 0.23367440700531006,
      "logps/chosen": -1.3996050357818604,
      "logps/rejected": -1.5623871088027954,
      "loss": 0.5572,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.3996050357818604,
      "rewards/margins": 0.16278214752674103,
      "rewards/rejected": -1.5623871088027954,
      "sft_loss": 1.3790435791015625,
      "step": 830
    },
    {
      "epoch": 0.44689747449406253,
      "grad_norm": 6.37550903604283,
      "learning_rate": 9.927337851142314e-07,
      "logits/chosen": 0.012400135397911072,
      "logits/rejected": 0.15300646424293518,
      "logps/chosen": -1.32003653049469,
      "logps/rejected": -1.5273606777191162,
      "loss": 0.5311,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.32003653049469,
      "rewards/margins": 0.20732417702674866,
      "rewards/rejected": -1.5273606777191162,
      "sft_loss": 1.3879741430282593,
      "step": 835
    },
    {
      "epoch": 0.44957350727546413,
      "grad_norm": 5.143220604120998,
      "learning_rate": 9.924668491496474e-07,
      "logits/chosen": 0.015577117912471294,
      "logits/rejected": 0.1794794499874115,
      "logps/chosen": -1.365195870399475,
      "logps/rejected": -1.6385002136230469,
      "loss": 0.5365,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.365195870399475,
      "rewards/margins": 0.27330437302589417,
      "rewards/rejected": -1.6385002136230469,
      "sft_loss": 1.4092786312103271,
      "step": 840
    },
    {
      "epoch": 0.4522495400568657,
      "grad_norm": 3.06536112732823,
      "learning_rate": 9.92195135269533e-07,
      "logits/chosen": 0.06246393918991089,
      "logits/rejected": 0.13810136914253235,
      "logps/chosen": -1.3615779876708984,
      "logps/rejected": -1.5043030977249146,
      "loss": 0.5668,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.3615779876708984,
      "rewards/margins": 0.14272502064704895,
      "rewards/rejected": -1.5043030977249146,
      "sft_loss": 1.4312576055526733,
      "step": 845
    },
    {
      "epoch": 0.4549255728382673,
      "grad_norm": 6.6171418934639314,
      "learning_rate": 9.919186461100574e-07,
      "logits/chosen": 0.017917241901159286,
      "logits/rejected": 0.09261620789766312,
      "logps/chosen": -1.3315399885177612,
      "logps/rejected": -1.5422439575195312,
      "loss": 0.5075,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3315399885177612,
      "rewards/margins": 0.21070384979248047,
      "rewards/rejected": -1.5422439575195312,
      "sft_loss": 1.3558510541915894,
      "step": 850
    },
    {
      "epoch": 0.45760160561966884,
      "grad_norm": 7.754700261996109,
      "learning_rate": 9.9163738435372e-07,
      "logits/chosen": -0.001633057021535933,
      "logits/rejected": 0.1511707454919815,
      "logps/chosen": -1.394895076751709,
      "logps/rejected": -1.6898698806762695,
      "loss": 0.5423,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.394895076751709,
      "rewards/margins": 0.29497483372688293,
      "rewards/rejected": -1.6898698806762695,
      "sft_loss": 1.4036824703216553,
      "step": 855
    },
    {
      "epoch": 0.4602776384010704,
      "grad_norm": 6.417177809723641,
      "learning_rate": 9.913513527293234e-07,
      "logits/chosen": -0.06009018421173096,
      "logits/rejected": 0.10860785096883774,
      "logps/chosen": -1.4335905313491821,
      "logps/rejected": -1.7497847080230713,
      "loss": 0.5269,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4335905313491821,
      "rewards/margins": 0.3161943256855011,
      "rewards/rejected": -1.7497847080230713,
      "sft_loss": 1.4489614963531494,
      "step": 860
    },
    {
      "epoch": 0.462953671182472,
      "grad_norm": 8.61348044658711,
      "learning_rate": 9.910605540119474e-07,
      "logits/chosen": 0.023756619542837143,
      "logits/rejected": 0.12181626260280609,
      "logps/chosen": -1.3452613353729248,
      "logps/rejected": -1.6940526962280273,
      "loss": 0.5213,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.3452613353729248,
      "rewards/margins": 0.3487912118434906,
      "rewards/rejected": -1.6940526962280273,
      "sft_loss": 1.3578007221221924,
      "step": 865
    },
    {
      "epoch": 0.46562970396387354,
      "grad_norm": 5.418686768484927,
      "learning_rate": 9.907649910229227e-07,
      "logits/chosen": -0.07046753168106079,
      "logits/rejected": 0.2019074410200119,
      "logps/chosen": -1.3916847705841064,
      "logps/rejected": -1.6394716501235962,
      "loss": 0.5432,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3916847705841064,
      "rewards/margins": 0.2477869987487793,
      "rewards/rejected": -1.6394716501235962,
      "sft_loss": 1.459414005279541,
      "step": 870
    },
    {
      "epoch": 0.46830573674527515,
      "grad_norm": 6.772815715405938,
      "learning_rate": 9.90464666629803e-07,
      "logits/chosen": 0.04510253667831421,
      "logits/rejected": 0.12748414278030396,
      "logps/chosen": -1.4449238777160645,
      "logps/rejected": -1.6484653949737549,
      "loss": 0.5854,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.4449238777160645,
      "rewards/margins": 0.20354151725769043,
      "rewards/rejected": -1.6484653949737549,
      "sft_loss": 1.4168418645858765,
      "step": 875
    },
    {
      "epoch": 0.4709817695266767,
      "grad_norm": 5.992740242144888,
      "learning_rate": 9.901595837463363e-07,
      "logits/chosen": 0.03951137140393257,
      "logits/rejected": 0.20961646735668182,
      "logps/chosen": -1.4726355075836182,
      "logps/rejected": -1.7111984491348267,
      "loss": 0.5574,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4726355075836182,
      "rewards/margins": 0.23856297135353088,
      "rewards/rejected": -1.7111984491348267,
      "sft_loss": 1.3962585926055908,
      "step": 880
    },
    {
      "epoch": 0.47365780230807825,
      "grad_norm": 5.702059239685596,
      "learning_rate": 9.898497453324384e-07,
      "logits/chosen": -0.026392418891191483,
      "logits/rejected": 0.058989159762859344,
      "logps/chosen": -1.3647552728652954,
      "logps/rejected": -1.648855209350586,
      "loss": 0.4968,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.3647552728652954,
      "rewards/margins": 0.2841000258922577,
      "rewards/rejected": -1.648855209350586,
      "sft_loss": 1.4022343158721924,
      "step": 885
    },
    {
      "epoch": 0.47633383508947985,
      "grad_norm": 4.659252230628318,
      "learning_rate": 9.895351543941628e-07,
      "logits/chosen": -0.13175614178180695,
      "logits/rejected": 0.0019244104623794556,
      "logps/chosen": -1.4099591970443726,
      "logps/rejected": -1.6276271343231201,
      "loss": 0.5377,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.4099591970443726,
      "rewards/margins": 0.21766802668571472,
      "rewards/rejected": -1.6276271343231201,
      "sft_loss": 1.463881492614746,
      "step": 890
    },
    {
      "epoch": 0.4790098678708814,
      "grad_norm": 4.770433623064993,
      "learning_rate": 9.892158139836724e-07,
      "logits/chosen": 0.07327640056610107,
      "logits/rejected": 0.19160141050815582,
      "logps/chosen": -1.283439040184021,
      "logps/rejected": -1.4736160039901733,
      "loss": 0.5258,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.283439040184021,
      "rewards/margins": 0.19017700850963593,
      "rewards/rejected": -1.4736160039901733,
      "sft_loss": 1.3432611227035522,
      "step": 895
    },
    {
      "epoch": 0.481685900652283,
      "grad_norm": 5.188465336691229,
      "learning_rate": 9.88891727199209e-07,
      "logits/chosen": -0.06001676991581917,
      "logits/rejected": 0.012325121089816093,
      "logps/chosen": -1.284280776977539,
      "logps/rejected": -1.5858594179153442,
      "loss": 0.5149,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.284280776977539,
      "rewards/margins": 0.30157846212387085,
      "rewards/rejected": -1.5858594179153442,
      "sft_loss": 1.3404700756072998,
      "step": 900
    },
    {
      "epoch": 0.48436193343368455,
      "grad_norm": 6.435494847996555,
      "learning_rate": 9.885628971850641e-07,
      "logits/chosen": 0.030720632523298264,
      "logits/rejected": 0.22497057914733887,
      "logps/chosen": -1.379372477531433,
      "logps/rejected": -1.6559817790985107,
      "loss": 0.5419,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.379372477531433,
      "rewards/margins": 0.2766092121601105,
      "rewards/rejected": -1.6559817790985107,
      "sft_loss": 1.4437005519866943,
      "step": 905
    },
    {
      "epoch": 0.48703796621508616,
      "grad_norm": 3.2066285057340056,
      "learning_rate": 9.882293271315481e-07,
      "logits/chosen": 0.007974756881594658,
      "logits/rejected": 0.11983414739370346,
      "logps/chosen": -1.4005982875823975,
      "logps/rejected": -1.572317123413086,
      "loss": 0.5646,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4005982875823975,
      "rewards/margins": 0.1717187911272049,
      "rewards/rejected": -1.572317123413086,
      "sft_loss": 1.3948160409927368,
      "step": 910
    },
    {
      "epoch": 0.4897139989964877,
      "grad_norm": 5.693222226682654,
      "learning_rate": 9.878910202749589e-07,
      "logits/chosen": 0.007499815430492163,
      "logits/rejected": 0.19532763957977295,
      "logps/chosen": -1.3502609729766846,
      "logps/rejected": -1.559653639793396,
      "loss": 0.5399,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3502609729766846,
      "rewards/margins": 0.20939283072948456,
      "rewards/rejected": -1.559653639793396,
      "sft_loss": 1.3668898344039917,
      "step": 915
    },
    {
      "epoch": 0.49239003177788926,
      "grad_norm": 6.159470735695498,
      "learning_rate": 9.875479798975512e-07,
      "logits/chosen": 0.14137479662895203,
      "logits/rejected": 0.287406325340271,
      "logps/chosen": -1.2880275249481201,
      "logps/rejected": -1.5608675479888916,
      "loss": 0.5292,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.2880275249481201,
      "rewards/margins": 0.2728400230407715,
      "rewards/rejected": -1.5608675479888916,
      "sft_loss": 1.352113962173462,
      "step": 920
    },
    {
      "epoch": 0.49506606455929086,
      "grad_norm": 6.255987887486548,
      "learning_rate": 9.87200209327504e-07,
      "logits/chosen": -0.013386614620685577,
      "logits/rejected": 0.1623094230890274,
      "logps/chosen": -1.4105921983718872,
      "logps/rejected": -1.5581471920013428,
      "loss": 0.5653,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.4105921983718872,
      "rewards/margins": 0.14755511283874512,
      "rewards/rejected": -1.5581471920013428,
      "sft_loss": 1.4051482677459717,
      "step": 925
    },
    {
      "epoch": 0.4977420973406924,
      "grad_norm": 8.388676102005673,
      "learning_rate": 9.868477119388894e-07,
      "logits/chosen": -0.0396624319255352,
      "logits/rejected": 0.07985645532608032,
      "logps/chosen": -1.3447215557098389,
      "logps/rejected": -1.6878904104232788,
      "loss": 0.5171,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3447215557098389,
      "rewards/margins": 0.34316885471343994,
      "rewards/rejected": -1.6878904104232788,
      "sft_loss": 1.3712944984436035,
      "step": 930
    },
    {
      "epoch": 0.500418130122094,
      "grad_norm": 4.908854695224659,
      "learning_rate": 9.864904911516383e-07,
      "logits/chosen": 0.05237286165356636,
      "logits/rejected": 0.09974393993616104,
      "logps/chosen": -1.3034316301345825,
      "logps/rejected": -1.5620901584625244,
      "loss": 0.5211,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3034316301345825,
      "rewards/margins": 0.25865834951400757,
      "rewards/rejected": -1.5620901584625244,
      "sft_loss": 1.3588597774505615,
      "step": 935
    },
    {
      "epoch": 0.5030941629034956,
      "grad_norm": 6.89034964552559,
      "learning_rate": 9.861285504315084e-07,
      "logits/chosen": 0.006044765003025532,
      "logits/rejected": 0.12427693605422974,
      "logps/chosen": -1.367592215538025,
      "logps/rejected": -1.5376479625701904,
      "loss": 0.5433,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.367592215538025,
      "rewards/margins": 0.17005568742752075,
      "rewards/rejected": -1.5376479625701904,
      "sft_loss": 1.4046542644500732,
      "step": 940
    },
    {
      "epoch": 0.5057701956848971,
      "grad_norm": 5.965890416950193,
      "learning_rate": 9.857618932900502e-07,
      "logits/chosen": -0.013211125507950783,
      "logits/rejected": 0.12869153916835785,
      "logps/chosen": -1.353560209274292,
      "logps/rejected": -1.6667423248291016,
      "loss": 0.5,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.353560209274292,
      "rewards/margins": 0.31318214535713196,
      "rewards/rejected": -1.6667423248291016,
      "sft_loss": 1.361161708831787,
      "step": 945
    },
    {
      "epoch": 0.5084462284662987,
      "grad_norm": 5.910356268986856,
      "learning_rate": 9.853905232845727e-07,
      "logits/chosen": -0.02266688272356987,
      "logits/rejected": 0.1527927666902542,
      "logps/chosen": -1.4592188596725464,
      "logps/rejected": -1.6313902139663696,
      "loss": 0.5854,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.4592188596725464,
      "rewards/margins": 0.17217124998569489,
      "rewards/rejected": -1.6313902139663696,
      "sft_loss": 1.4636166095733643,
      "step": 950
    },
    {
      "epoch": 0.5111222612477003,
      "grad_norm": 5.828938279226205,
      "learning_rate": 9.850144440181095e-07,
      "logits/chosen": 0.025441814213991165,
      "logits/rejected": 0.2594681978225708,
      "logps/chosen": -1.4633724689483643,
      "logps/rejected": -1.6635366678237915,
      "loss": 0.5688,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4633724689483643,
      "rewards/margins": 0.20016424357891083,
      "rewards/rejected": -1.6635366678237915,
      "sft_loss": 1.5087789297103882,
      "step": 955
    },
    {
      "epoch": 0.5137982940291018,
      "grad_norm": 6.916444373102311,
      "learning_rate": 9.846336591393832e-07,
      "logits/chosen": -0.019559945911169052,
      "logits/rejected": 0.13119149208068848,
      "logps/chosen": -1.3910858631134033,
      "logps/rejected": -1.5714839696884155,
      "loss": 0.5552,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3910858631134033,
      "rewards/margins": 0.1803981512784958,
      "rewards/rejected": -1.5714839696884155,
      "sft_loss": 1.4235373735427856,
      "step": 960
    },
    {
      "epoch": 0.5164743268105034,
      "grad_norm": 5.783815333575644,
      "learning_rate": 9.842481723427704e-07,
      "logits/chosen": 0.06887931376695633,
      "logits/rejected": 0.0669599249958992,
      "logps/chosen": -1.4362841844558716,
      "logps/rejected": -1.7442407608032227,
      "loss": 0.5408,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4362841844558716,
      "rewards/margins": 0.3079567551612854,
      "rewards/rejected": -1.7442407608032227,
      "sft_loss": 1.4661349058151245,
      "step": 965
    },
    {
      "epoch": 0.519150359591905,
      "grad_norm": 5.002807863509631,
      "learning_rate": 9.838579873682658e-07,
      "logits/chosen": 0.09101974219083786,
      "logits/rejected": 0.10050985962152481,
      "logps/chosen": -1.3150116205215454,
      "logps/rejected": -1.5524133443832397,
      "loss": 0.5242,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3150116205215454,
      "rewards/margins": 0.23740187287330627,
      "rewards/rejected": -1.5524133443832397,
      "sft_loss": 1.3296570777893066,
      "step": 970
    },
    {
      "epoch": 0.5218263923733065,
      "grad_norm": 5.278186807526357,
      "learning_rate": 9.834631080014457e-07,
      "logits/chosen": -0.05638844892382622,
      "logits/rejected": 0.14079150557518005,
      "logps/chosen": -1.3863670825958252,
      "logps/rejected": -1.5993766784667969,
      "loss": 0.5245,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3863670825958252,
      "rewards/margins": 0.21300962567329407,
      "rewards/rejected": -1.5993766784667969,
      "sft_loss": 1.4488608837127686,
      "step": 975
    },
    {
      "epoch": 0.5245024251547081,
      "grad_norm": 7.842449217425358,
      "learning_rate": 9.830635380734312e-07,
      "logits/chosen": -0.04868556931614876,
      "logits/rejected": 0.15271759033203125,
      "logps/chosen": -1.4653922319412231,
      "logps/rejected": -1.6876140832901,
      "loss": 0.5627,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4653922319412231,
      "rewards/margins": 0.22222192585468292,
      "rewards/rejected": -1.6876140832901,
      "sft_loss": 1.4675097465515137,
      "step": 980
    },
    {
      "epoch": 0.5271784579361097,
      "grad_norm": 6.6791345392995725,
      "learning_rate": 9.826592814608517e-07,
      "logits/chosen": 0.03832697868347168,
      "logits/rejected": 0.23097598552703857,
      "logps/chosen": -1.4058834314346313,
      "logps/rejected": -1.648074746131897,
      "loss": 0.5376,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4058834314346313,
      "rewards/margins": 0.24219119548797607,
      "rewards/rejected": -1.648074746131897,
      "sft_loss": 1.4827638864517212,
      "step": 985
    },
    {
      "epoch": 0.5298544907175113,
      "grad_norm": 4.400687968516655,
      "learning_rate": 9.822503420858067e-07,
      "logits/chosen": 0.07663445919752121,
      "logits/rejected": 0.11500799655914307,
      "logps/chosen": -1.2576382160186768,
      "logps/rejected": -1.5772205591201782,
      "loss": 0.4804,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.2576382160186768,
      "rewards/margins": 0.3195821940898895,
      "rewards/rejected": -1.5772205591201782,
      "sft_loss": 1.3516136407852173,
      "step": 990
    },
    {
      "epoch": 0.5325305234989128,
      "grad_norm": 5.496637559141165,
      "learning_rate": 9.818367239158277e-07,
      "logits/chosen": 0.12294967472553253,
      "logits/rejected": 0.209446981549263,
      "logps/chosen": -1.3757258653640747,
      "logps/rejected": -1.53915274143219,
      "loss": 0.5722,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.3757258653640747,
      "rewards/margins": 0.16342684626579285,
      "rewards/rejected": -1.53915274143219,
      "sft_loss": 1.4503815174102783,
      "step": 995
    },
    {
      "epoch": 0.5352065562803144,
      "grad_norm": 7.084586699245085,
      "learning_rate": 9.8141843096384e-07,
      "logits/chosen": 0.11083235591650009,
      "logits/rejected": 0.2419196367263794,
      "logps/chosen": -1.403327465057373,
      "logps/rejected": -1.692497968673706,
      "loss": 0.5508,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.403327465057373,
      "rewards/margins": 0.2891707420349121,
      "rewards/rejected": -1.692497968673706,
      "sft_loss": 1.4336564540863037,
      "step": 1000
    },
    {
      "epoch": 0.537882589061716,
      "grad_norm": 8.470327477689404,
      "learning_rate": 9.809954672881237e-07,
      "logits/chosen": 0.06831010431051254,
      "logits/rejected": 0.24153780937194824,
      "logps/chosen": -1.4360759258270264,
      "logps/rejected": -1.7128865718841553,
      "loss": 0.5464,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4360759258270264,
      "rewards/margins": 0.2768106460571289,
      "rewards/rejected": -1.7128865718841553,
      "sft_loss": 1.484975814819336,
      "step": 1005
    },
    {
      "epoch": 0.5405586218431175,
      "grad_norm": 5.585928773503728,
      "learning_rate": 9.80567836992274e-07,
      "logits/chosen": 0.017606690526008606,
      "logits/rejected": 0.2122047394514084,
      "logps/chosen": -1.2979730367660522,
      "logps/rejected": -1.6680152416229248,
      "loss": 0.4977,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.2979730367660522,
      "rewards/margins": 0.37004226446151733,
      "rewards/rejected": -1.6680152416229248,
      "sft_loss": 1.3362150192260742,
      "step": 1010
    },
    {
      "epoch": 0.5432346546245191,
      "grad_norm": 6.0789178538079325,
      "learning_rate": 9.801355442251625e-07,
      "logits/chosen": 0.031071409583091736,
      "logits/rejected": 0.21970124542713165,
      "logps/chosen": -1.351664662361145,
      "logps/rejected": -1.6419140100479126,
      "loss": 0.514,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.351664662361145,
      "rewards/margins": 0.29024919867515564,
      "rewards/rejected": -1.6419140100479126,
      "sft_loss": 1.4079734086990356,
      "step": 1015
    },
    {
      "epoch": 0.5459106874059207,
      "grad_norm": 6.839970819560228,
      "learning_rate": 9.796985931808949e-07,
      "logits/chosen": 0.038407161831855774,
      "logits/rejected": 0.1905667930841446,
      "logps/chosen": -1.4028685092926025,
      "logps/rejected": -1.7034860849380493,
      "loss": 0.5103,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4028685092926025,
      "rewards/margins": 0.300617516040802,
      "rewards/rejected": -1.7034860849380493,
      "sft_loss": 1.4510376453399658,
      "step": 1020
    },
    {
      "epoch": 0.5485867201873222,
      "grad_norm": 6.261514936310619,
      "learning_rate": 9.792569880987724e-07,
      "logits/chosen": -0.0038838565815240145,
      "logits/rejected": 0.12509891390800476,
      "logps/chosen": -1.3250789642333984,
      "logps/rejected": -1.6913788318634033,
      "loss": 0.504,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3250789642333984,
      "rewards/margins": 0.3662997782230377,
      "rewards/rejected": -1.6913788318634033,
      "sft_loss": 1.3589133024215698,
      "step": 1025
    },
    {
      "epoch": 0.5512627529687238,
      "grad_norm": 7.532636921841018,
      "learning_rate": 9.788107332632493e-07,
      "logits/chosen": 0.021292533725500107,
      "logits/rejected": 0.11308477818965912,
      "logps/chosen": -1.3988656997680664,
      "logps/rejected": -1.6569464206695557,
      "loss": 0.5384,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3988656997680664,
      "rewards/margins": 0.2580806612968445,
      "rewards/rejected": -1.6569464206695557,
      "sft_loss": 1.4494415521621704,
      "step": 1030
    },
    {
      "epoch": 0.5539387857501255,
      "grad_norm": 7.267928662402311,
      "learning_rate": 9.783598330038924e-07,
      "logits/chosen": -0.0075324522331357,
      "logits/rejected": 0.11378462612628937,
      "logps/chosen": -1.455899953842163,
      "logps/rejected": -1.6032978296279907,
      "loss": 0.5654,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.455899953842163,
      "rewards/margins": 0.14739762246608734,
      "rewards/rejected": -1.6032978296279907,
      "sft_loss": 1.462633490562439,
      "step": 1035
    },
    {
      "epoch": 0.5566148185315271,
      "grad_norm": 9.270113397343684,
      "learning_rate": 9.779042916953376e-07,
      "logits/chosen": 0.03945959731936455,
      "logits/rejected": 0.20420022308826447,
      "logps/chosen": -1.355939269065857,
      "logps/rejected": -1.6491836309432983,
      "loss": 0.5248,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.355939269065857,
      "rewards/margins": 0.2932443618774414,
      "rewards/rejected": -1.6491836309432983,
      "sft_loss": 1.4178965091705322,
      "step": 1040
    },
    {
      "epoch": 0.5592908513129285,
      "grad_norm": 3.9428774371292308,
      "learning_rate": 9.774441137572487e-07,
      "logits/chosen": -0.029956543818116188,
      "logits/rejected": 0.1199512928724289,
      "logps/chosen": -1.3319822549819946,
      "logps/rejected": -1.6581604480743408,
      "loss": 0.5112,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.3319822549819946,
      "rewards/margins": 0.32617828249931335,
      "rewards/rejected": -1.6581604480743408,
      "sft_loss": 1.4374191761016846,
      "step": 1045
    },
    {
      "epoch": 0.5619668840943302,
      "grad_norm": 8.900796310027207,
      "learning_rate": 9.76979303654274e-07,
      "logits/chosen": -0.08117417246103287,
      "logits/rejected": 0.0267164446413517,
      "logps/chosen": -1.424697756767273,
      "logps/rejected": -1.7513818740844727,
      "loss": 0.5297,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.424697756767273,
      "rewards/margins": 0.32668399810791016,
      "rewards/rejected": -1.7513818740844727,
      "sft_loss": 1.4675133228302002,
      "step": 1050
    },
    {
      "epoch": 0.5646429168757318,
      "grad_norm": 8.31046841086775,
      "learning_rate": 9.765098658960035e-07,
      "logits/chosen": 0.008648221381008625,
      "logits/rejected": 0.09285394847393036,
      "logps/chosen": -1.428062081336975,
      "logps/rejected": -1.715243935585022,
      "loss": 0.5195,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.428062081336975,
      "rewards/margins": 0.2871818542480469,
      "rewards/rejected": -1.715243935585022,
      "sft_loss": 1.459547758102417,
      "step": 1055
    },
    {
      "epoch": 0.5673189496571333,
      "grad_norm": 8.420046513363681,
      "learning_rate": 9.76035805036924e-07,
      "logits/chosen": 0.03144007921218872,
      "logits/rejected": 0.2124161720275879,
      "logps/chosen": -1.5106102228164673,
      "logps/rejected": -1.763193130493164,
      "loss": 0.5501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5106102228164673,
      "rewards/margins": 0.25258293747901917,
      "rewards/rejected": -1.763193130493164,
      "sft_loss": 1.495023488998413,
      "step": 1060
    },
    {
      "epoch": 0.5699949824385349,
      "grad_norm": 3.7418036331803304,
      "learning_rate": 9.755571256763764e-07,
      "logits/chosen": 0.047572363168001175,
      "logits/rejected": 0.18473589420318604,
      "logps/chosen": -1.4035441875457764,
      "logps/rejected": -1.80259108543396,
      "loss": 0.4896,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4035441875457764,
      "rewards/margins": 0.3990468978881836,
      "rewards/rejected": -1.80259108543396,
      "sft_loss": 1.4959805011749268,
      "step": 1065
    },
    {
      "epoch": 0.5726710152199365,
      "grad_norm": 5.7848942867266695,
      "learning_rate": 9.750738324585097e-07,
      "logits/chosen": -0.08872415125370026,
      "logits/rejected": 0.15845569968223572,
      "logps/chosen": -1.456129789352417,
      "logps/rejected": -1.7990448474884033,
      "loss": 0.5193,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.456129789352417,
      "rewards/margins": 0.34291499853134155,
      "rewards/rejected": -1.7990448474884033,
      "sft_loss": 1.4630060195922852,
      "step": 1070
    },
    {
      "epoch": 0.5753470480013381,
      "grad_norm": 4.524211282548457,
      "learning_rate": 9.74585930072237e-07,
      "logits/chosen": -0.018144235014915466,
      "logits/rejected": 0.11269550025463104,
      "logps/chosen": -1.3974212408065796,
      "logps/rejected": -1.774336576461792,
      "loss": 0.5062,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.3974212408065796,
      "rewards/margins": 0.3769153952598572,
      "rewards/rejected": -1.774336576461792,
      "sft_loss": 1.4539904594421387,
      "step": 1075
    },
    {
      "epoch": 0.5780230807827396,
      "grad_norm": 6.618978905317209,
      "learning_rate": 9.740934232511892e-07,
      "logits/chosen": -0.06820371747016907,
      "logits/rejected": 0.0417419895529747,
      "logps/chosen": -1.5021356344223022,
      "logps/rejected": -1.7642343044281006,
      "loss": 0.5427,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.5021356344223022,
      "rewards/margins": 0.26209884881973267,
      "rewards/rejected": -1.7642343044281006,
      "sft_loss": 1.5417989492416382,
      "step": 1080
    },
    {
      "epoch": 0.5806991135641412,
      "grad_norm": 8.575313314221772,
      "learning_rate": 9.735963167736698e-07,
      "logits/chosen": -0.010738177224993706,
      "logits/rejected": 0.15471696853637695,
      "logps/chosen": -1.4406782388687134,
      "logps/rejected": -1.554862141609192,
      "loss": 0.5903,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.4406782388687134,
      "rewards/margins": 0.11418372392654419,
      "rewards/rejected": -1.554862141609192,
      "sft_loss": 1.4563452005386353,
      "step": 1085
    },
    {
      "epoch": 0.5833751463455428,
      "grad_norm": 6.240052893582782,
      "learning_rate": 9.730946154626078e-07,
      "logits/chosen": -0.0038662850856781006,
      "logits/rejected": 0.10398884862661362,
      "logps/chosen": -1.40817129611969,
      "logps/rejected": -1.5353500843048096,
      "loss": 0.571,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.40817129611969,
      "rewards/margins": 0.1271788775920868,
      "rewards/rejected": -1.5353500843048096,
      "sft_loss": 1.4028632640838623,
      "step": 1090
    },
    {
      "epoch": 0.5860511791269443,
      "grad_norm": 5.2462033549546865,
      "learning_rate": 9.725883241855117e-07,
      "logits/chosen": -0.13482658565044403,
      "logits/rejected": -0.0035198740661144257,
      "logps/chosen": -1.3518860340118408,
      "logps/rejected": -1.6204659938812256,
      "loss": 0.5088,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3518860340118408,
      "rewards/margins": 0.26857990026474,
      "rewards/rejected": -1.6204659938812256,
      "sft_loss": 1.4049688577651978,
      "step": 1095
    },
    {
      "epoch": 0.5887272119083459,
      "grad_norm": 5.863844446116353,
      "learning_rate": 9.720774478544218e-07,
      "logits/chosen": 0.015030590817332268,
      "logits/rejected": 0.11544246971607208,
      "logps/chosen": -1.2636845111846924,
      "logps/rejected": -1.7326905727386475,
      "loss": 0.4714,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.2636845111846924,
      "rewards/margins": 0.46900612115859985,
      "rewards/rejected": -1.7326905727386475,
      "sft_loss": 1.3195103406906128,
      "step": 1100
    },
    {
      "epoch": 0.5914032446897475,
      "grad_norm": 4.594870990121541,
      "learning_rate": 9.715619914258624e-07,
      "logits/chosen": -0.06224752590060234,
      "logits/rejected": 0.020744603127241135,
      "logps/chosen": -1.381974220275879,
      "logps/rejected": -1.5955941677093506,
      "loss": 0.5413,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.381974220275879,
      "rewards/margins": 0.21361997723579407,
      "rewards/rejected": -1.5955941677093506,
      "sft_loss": 1.3867851495742798,
      "step": 1105
    },
    {
      "epoch": 0.594079277471149,
      "grad_norm": 6.092708307677853,
      "learning_rate": 9.710419599007937e-07,
      "logits/chosen": -0.04417000710964203,
      "logits/rejected": 0.07652486860752106,
      "logps/chosen": -1.3594367504119873,
      "logps/rejected": -1.519745111465454,
      "loss": 0.5404,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.3594367504119873,
      "rewards/margins": 0.16030827164649963,
      "rewards/rejected": -1.519745111465454,
      "sft_loss": 1.3841520547866821,
      "step": 1110
    },
    {
      "epoch": 0.5967553102525506,
      "grad_norm": 10.943720385690531,
      "learning_rate": 9.705173583245643e-07,
      "logits/chosen": 0.029814088717103004,
      "logits/rejected": 0.1491873860359192,
      "logps/chosen": -1.2818458080291748,
      "logps/rejected": -1.5984141826629639,
      "loss": 0.4868,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.2818458080291748,
      "rewards/margins": 0.31656843423843384,
      "rewards/rejected": -1.5984141826629639,
      "sft_loss": 1.2908055782318115,
      "step": 1115
    },
    {
      "epoch": 0.5994313430339522,
      "grad_norm": 5.5118942012596515,
      "learning_rate": 9.699881917868609e-07,
      "logits/chosen": -0.15234415233135223,
      "logits/rejected": -0.04955081641674042,
      "logps/chosen": -1.3449537754058838,
      "logps/rejected": -1.6491496562957764,
      "loss": 0.5132,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3449537754058838,
      "rewards/margins": 0.304195761680603,
      "rewards/rejected": -1.6491496562957764,
      "sft_loss": 1.3979218006134033,
      "step": 1120
    },
    {
      "epoch": 0.6021073758153538,
      "grad_norm": 8.021170292513808,
      "learning_rate": 9.694544654216594e-07,
      "logits/chosen": -0.15768754482269287,
      "logits/rejected": 0.024920865893363953,
      "logps/chosen": -1.4138802289962769,
      "logps/rejected": -1.7606861591339111,
      "loss": 0.5116,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.4138802289962769,
      "rewards/margins": 0.3468059003353119,
      "rewards/rejected": -1.7606861591339111,
      "sft_loss": 1.4192440509796143,
      "step": 1125
    },
    {
      "epoch": 0.6047834085967553,
      "grad_norm": 6.295529086087212,
      "learning_rate": 9.689161844071755e-07,
      "logits/chosen": 0.013290470466017723,
      "logits/rejected": 0.06877493858337402,
      "logps/chosen": -1.4313960075378418,
      "logps/rejected": -1.7183454036712646,
      "loss": 0.5347,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4313960075378418,
      "rewards/margins": 0.2869493067264557,
      "rewards/rejected": -1.7183454036712646,
      "sft_loss": 1.3810927867889404,
      "step": 1130
    },
    {
      "epoch": 0.6074594413781569,
      "grad_norm": 8.859671073714802,
      "learning_rate": 9.683733539658138e-07,
      "logits/chosen": -0.06083168461918831,
      "logits/rejected": 0.09201276302337646,
      "logps/chosen": -1.5324196815490723,
      "logps/rejected": -1.8598239421844482,
      "loss": 0.5551,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5324196815490723,
      "rewards/margins": 0.3274044394493103,
      "rewards/rejected": -1.8598239421844482,
      "sft_loss": 1.4262425899505615,
      "step": 1135
    },
    {
      "epoch": 0.6101354741595585,
      "grad_norm": 7.030273446229885,
      "learning_rate": 9.678259793641178e-07,
      "logits/chosen": -0.06408004462718964,
      "logits/rejected": -0.016680490225553513,
      "logps/chosen": -1.4756966829299927,
      "logps/rejected": -1.6183178424835205,
      "loss": 0.5804,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4756966829299927,
      "rewards/margins": 0.1426212340593338,
      "rewards/rejected": -1.6183178424835205,
      "sft_loss": 1.5306628942489624,
      "step": 1140
    },
    {
      "epoch": 0.61281150694096,
      "grad_norm": 5.8221650590700404,
      "learning_rate": 9.672740659127183e-07,
      "logits/chosen": -0.18864266574382782,
      "logits/rejected": -0.0647595077753067,
      "logps/chosen": -1.4422852993011475,
      "logps/rejected": -1.7732452154159546,
      "loss": 0.5433,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4422852993011475,
      "rewards/margins": 0.330960214138031,
      "rewards/rejected": -1.7732452154159546,
      "sft_loss": 1.5205518007278442,
      "step": 1145
    },
    {
      "epoch": 0.6154875397223616,
      "grad_norm": 5.871581315934256,
      "learning_rate": 9.667176189662818e-07,
      "logits/chosen": -0.16291369497776031,
      "logits/rejected": -0.028021007776260376,
      "logps/chosen": -1.3359414339065552,
      "logps/rejected": -1.6861404180526733,
      "loss": 0.4926,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3359414339065552,
      "rewards/margins": 0.3501989245414734,
      "rewards/rejected": -1.6861404180526733,
      "sft_loss": 1.3437201976776123,
      "step": 1150
    },
    {
      "epoch": 0.6181635725037632,
      "grad_norm": 5.387313145530907,
      "learning_rate": 9.661566439234592e-07,
      "logits/chosen": -0.07084006071090698,
      "logits/rejected": 0.022573301568627357,
      "logps/chosen": -1.4162758588790894,
      "logps/rejected": -1.6274665594100952,
      "loss": 0.546,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4162758588790894,
      "rewards/margins": 0.21119055151939392,
      "rewards/rejected": -1.6274665594100952,
      "sft_loss": 1.4579969644546509,
      "step": 1155
    },
    {
      "epoch": 0.6208396052851648,
      "grad_norm": 6.915753400263389,
      "learning_rate": 9.655911462268327e-07,
      "logits/chosen": -0.010980304330587387,
      "logits/rejected": 0.08265860378742218,
      "logps/chosen": -1.3437680006027222,
      "logps/rejected": -1.6253420114517212,
      "loss": 0.5002,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3437680006027222,
      "rewards/margins": 0.28157392144203186,
      "rewards/rejected": -1.6253420114517212,
      "sft_loss": 1.4295752048492432,
      "step": 1160
    },
    {
      "epoch": 0.6235156380665663,
      "grad_norm": 4.3569418482421955,
      "learning_rate": 9.650211313628636e-07,
      "logits/chosen": -0.06969741731882095,
      "logits/rejected": 0.010083474218845367,
      "logps/chosen": -1.3329765796661377,
      "logps/rejected": -1.5761654376983643,
      "loss": 0.5134,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3329765796661377,
      "rewards/margins": 0.2431887835264206,
      "rewards/rejected": -1.5761654376983643,
      "sft_loss": 1.3583405017852783,
      "step": 1165
    },
    {
      "epoch": 0.6261916708479679,
      "grad_norm": 4.473590718113074,
      "learning_rate": 9.644466048618386e-07,
      "logits/chosen": -0.08624277263879776,
      "logits/rejected": 0.07240621745586395,
      "logps/chosen": -1.5664074420928955,
      "logps/rejected": -1.7688283920288086,
      "loss": 0.5873,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.5664074420928955,
      "rewards/margins": 0.20242106914520264,
      "rewards/rejected": -1.7688283920288086,
      "sft_loss": 1.532112717628479,
      "step": 1170
    },
    {
      "epoch": 0.6288677036293695,
      "grad_norm": 4.114132484032189,
      "learning_rate": 9.63867572297816e-07,
      "logits/chosen": -0.07057038694620132,
      "logits/rejected": 0.10959811508655548,
      "logps/chosen": -1.3659435510635376,
      "logps/rejected": -1.6666004657745361,
      "loss": 0.5277,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3659435510635376,
      "rewards/margins": 0.30065685510635376,
      "rewards/rejected": -1.6666004657745361,
      "sft_loss": 1.4245878458023071,
      "step": 1175
    },
    {
      "epoch": 0.631543736410771,
      "grad_norm": 5.164074151299386,
      "learning_rate": 9.632840392885727e-07,
      "logits/chosen": -0.08701647818088531,
      "logits/rejected": 0.05283905193209648,
      "logps/chosen": -1.4877811670303345,
      "logps/rejected": -1.8170543909072876,
      "loss": 0.5181,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4877811670303345,
      "rewards/margins": 0.3292733430862427,
      "rewards/rejected": -1.8170543909072876,
      "sft_loss": 1.4720274209976196,
      "step": 1180
    },
    {
      "epoch": 0.6342197691921726,
      "grad_norm": 7.34774728696768,
      "learning_rate": 9.626960114955483e-07,
      "logits/chosen": -0.017324324697256088,
      "logits/rejected": 0.12090086936950684,
      "logps/chosen": -1.5208476781845093,
      "logps/rejected": -1.7934768199920654,
      "loss": 0.5548,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.5208476781845093,
      "rewards/margins": 0.2726292014122009,
      "rewards/rejected": -1.7934768199920654,
      "sft_loss": 1.473417043685913,
      "step": 1185
    },
    {
      "epoch": 0.6368958019735742,
      "grad_norm": 6.258376100266322,
      "learning_rate": 9.621034946237909e-07,
      "logits/chosen": -0.0985368937253952,
      "logits/rejected": 0.04786267876625061,
      "logps/chosen": -1.4638845920562744,
      "logps/rejected": -1.8589398860931396,
      "loss": 0.4907,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4638845920562744,
      "rewards/margins": 0.39505526423454285,
      "rewards/rejected": -1.8589398860931396,
      "sft_loss": 1.4805598258972168,
      "step": 1190
    },
    {
      "epoch": 0.6395718347549757,
      "grad_norm": 4.5357941902003915,
      "learning_rate": 9.615064944219021e-07,
      "logits/chosen": -0.04397290572524071,
      "logits/rejected": 0.07348016649484634,
      "logps/chosen": -1.344813346862793,
      "logps/rejected": -1.7454315423965454,
      "loss": 0.4868,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.344813346862793,
      "rewards/margins": 0.4006180763244629,
      "rewards/rejected": -1.7454315423965454,
      "sft_loss": 1.427937626838684,
      "step": 1195
    },
    {
      "epoch": 0.6422478675363773,
      "grad_norm": 9.259125217654567,
      "learning_rate": 9.609050166819803e-07,
      "logits/chosen": -0.11931729316711426,
      "logits/rejected": -0.050427474081516266,
      "logps/chosen": -1.4302603006362915,
      "logps/rejected": -1.7228256464004517,
      "loss": 0.5415,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4302603006362915,
      "rewards/margins": 0.2925655245780945,
      "rewards/rejected": -1.7228256464004517,
      "sft_loss": 1.433077096939087,
      "step": 1200
    },
    {
      "epoch": 0.6422478675363773,
      "eval_logits/chosen": 0.2312067151069641,
      "eval_logits/rejected": 0.32195788621902466,
      "eval_logps/chosen": -1.439756155014038,
      "eval_logps/rejected": -1.789048194885254,
      "eval_loss": 0.5156100392341614,
      "eval_rewards/accuracies": 0.6186943650245667,
      "eval_rewards/chosen": -1.439756155014038,
      "eval_rewards/margins": 0.349292129278183,
      "eval_rewards/rejected": -1.789048194885254,
      "eval_runtime": 43.2063,
      "eval_samples_per_second": 31.13,
      "eval_sft_loss": 1.4648340940475464,
      "eval_steps_per_second": 7.8,
      "step": 1200
    },
    {
      "epoch": 0.6449239003177789,
      "grad_norm": 7.497624604633386,
      "learning_rate": 9.602990672395653e-07,
      "logits/chosen": -0.2047913521528244,
      "logits/rejected": -0.028477761894464493,
      "logps/chosen": -1.4075069427490234,
      "logps/rejected": -1.695677399635315,
      "loss": 0.5239,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4075069427490234,
      "rewards/margins": 0.28817063570022583,
      "rewards/rejected": -1.695677399635315,
      "sft_loss": 1.457287073135376,
      "step": 1205
    },
    {
      "epoch": 0.6475999330991805,
      "grad_norm": 5.828011221905981,
      "learning_rate": 9.59688651973581e-07,
      "logits/chosen": -0.10706796497106552,
      "logits/rejected": 0.09093932807445526,
      "logps/chosen": -1.3917760848999023,
      "logps/rejected": -1.6692960262298584,
      "loss": 0.5134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3917760848999023,
      "rewards/margins": 0.2775200605392456,
      "rewards/rejected": -1.6692960262298584,
      "sft_loss": 1.4143939018249512,
      "step": 1210
    },
    {
      "epoch": 0.650275965880582,
      "grad_norm": 4.494440416665394,
      "learning_rate": 9.590737768062792e-07,
      "logits/chosen": -0.13113786280155182,
      "logits/rejected": -0.008872026577591896,
      "logps/chosen": -1.412584900856018,
      "logps/rejected": -1.5937589406967163,
      "loss": 0.5596,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.412584900856018,
      "rewards/margins": 0.1811741143465042,
      "rewards/rejected": -1.5937589406967163,
      "sft_loss": 1.4540883302688599,
      "step": 1215
    },
    {
      "epoch": 0.6529519986619836,
      "grad_norm": 6.339627030901809,
      "learning_rate": 9.584544477031816e-07,
      "logits/chosen": 0.03890204429626465,
      "logits/rejected": 0.1465032696723938,
      "logps/chosen": -1.3235418796539307,
      "logps/rejected": -1.5791696310043335,
      "loss": 0.5228,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3235418796539307,
      "rewards/margins": 0.25562790036201477,
      "rewards/rejected": -1.5791696310043335,
      "sft_loss": 1.3354781866073608,
      "step": 1220
    },
    {
      "epoch": 0.6556280314433852,
      "grad_norm": 5.280026773959482,
      "learning_rate": 9.578306706730215e-07,
      "logits/chosen": -0.15729853510856628,
      "logits/rejected": 0.05795856565237045,
      "logps/chosen": -1.4452846050262451,
      "logps/rejected": -1.6697638034820557,
      "loss": 0.5469,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4452846050262451,
      "rewards/margins": 0.224479079246521,
      "rewards/rejected": -1.6697638034820557,
      "sft_loss": 1.487400770187378,
      "step": 1225
    },
    {
      "epoch": 0.6583040642247867,
      "grad_norm": 6.4958026268078966,
      "learning_rate": 9.572024517676865e-07,
      "logits/chosen": -0.054301343858242035,
      "logits/rejected": 0.048134349286556244,
      "logps/chosen": -1.3963154554367065,
      "logps/rejected": -1.656686782836914,
      "loss": 0.5308,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3963154554367065,
      "rewards/margins": 0.2603713870048523,
      "rewards/rejected": -1.656686782836914,
      "sft_loss": 1.3829001188278198,
      "step": 1230
    },
    {
      "epoch": 0.6609800970061883,
      "grad_norm": 4.727866949980633,
      "learning_rate": 9.565697970821593e-07,
      "logits/chosen": -0.01401115395128727,
      "logits/rejected": 0.12580223381519318,
      "logps/chosen": -1.4165058135986328,
      "logps/rejected": -1.6185691356658936,
      "loss": 0.5372,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4165058135986328,
      "rewards/margins": 0.20206335186958313,
      "rewards/rejected": -1.6185691356658936,
      "sft_loss": 1.5026344060897827,
      "step": 1235
    },
    {
      "epoch": 0.6636561297875899,
      "grad_norm": 5.596557785835436,
      "learning_rate": 9.559327127544585e-07,
      "logits/chosen": -0.16433796286582947,
      "logits/rejected": -0.03051048144698143,
      "logps/chosen": -1.3875510692596436,
      "logps/rejected": -1.6686878204345703,
      "loss": 0.5052,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3875510692596436,
      "rewards/margins": 0.2811369299888611,
      "rewards/rejected": -1.6686878204345703,
      "sft_loss": 1.4683420658111572,
      "step": 1240
    },
    {
      "epoch": 0.6663321625689914,
      "grad_norm": 5.21063206823352,
      "learning_rate": 9.552912049655789e-07,
      "logits/chosen": -0.06284203380346298,
      "logits/rejected": 0.12157057225704193,
      "logps/chosen": -1.4473823308944702,
      "logps/rejected": -1.6761335134506226,
      "loss": 0.5671,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.4473823308944702,
      "rewards/margins": 0.22875113785266876,
      "rewards/rejected": -1.6761335134506226,
      "sft_loss": 1.5018178224563599,
      "step": 1245
    },
    {
      "epoch": 0.669008195350393,
      "grad_norm": 8.033877889431182,
      "learning_rate": 9.546452799394315e-07,
      "logits/chosen": -0.06265360862016678,
      "logits/rejected": 0.14023162424564362,
      "logps/chosen": -1.5192325115203857,
      "logps/rejected": -1.736997365951538,
      "loss": 0.5536,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.5192325115203857,
      "rewards/margins": 0.21776482462882996,
      "rewards/rejected": -1.736997365951538,
      "sft_loss": 1.5233339071273804,
      "step": 1250
    },
    {
      "epoch": 0.6716842281317946,
      "grad_norm": 7.9555308238631905,
      "learning_rate": 9.539949439427846e-07,
      "logits/chosen": -0.060469646006822586,
      "logits/rejected": 0.07171236723661423,
      "logps/chosen": -1.4047214984893799,
      "logps/rejected": -1.686340093612671,
      "loss": 0.5278,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4047214984893799,
      "rewards/margins": 0.28161847591400146,
      "rewards/rejected": -1.686340093612671,
      "sft_loss": 1.4950298070907593,
      "step": 1255
    },
    {
      "epoch": 0.6743602609131962,
      "grad_norm": 4.922792255561007,
      "learning_rate": 9.533402032852002e-07,
      "logits/chosen": -0.09880800545215607,
      "logits/rejected": 0.03451662138104439,
      "logps/chosen": -1.399361491203308,
      "logps/rejected": -1.7778685092926025,
      "loss": 0.5055,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.399361491203308,
      "rewards/margins": 0.3785068094730377,
      "rewards/rejected": -1.7778685092926025,
      "sft_loss": 1.468930959701538,
      "step": 1260
    },
    {
      "epoch": 0.6770362936945977,
      "grad_norm": 6.19723798522441,
      "learning_rate": 9.526810643189754e-07,
      "logits/chosen": 0.0034088001120835543,
      "logits/rejected": 0.1572073996067047,
      "logps/chosen": -1.4213447570800781,
      "logps/rejected": -1.7516653537750244,
      "loss": 0.5194,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4213447570800781,
      "rewards/margins": 0.33032044768333435,
      "rewards/rejected": -1.7516653537750244,
      "sft_loss": 1.4661481380462646,
      "step": 1265
    },
    {
      "epoch": 0.6797123264759993,
      "grad_norm": 6.050036047925458,
      "learning_rate": 9.52017533439079e-07,
      "logits/chosen": -0.09415547549724579,
      "logits/rejected": 0.01310119591653347,
      "logps/chosen": -1.4122450351715088,
      "logps/rejected": -1.7994779348373413,
      "loss": 0.5066,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4122450351715088,
      "rewards/margins": 0.3872327208518982,
      "rewards/rejected": -1.7994779348373413,
      "sft_loss": 1.4726154804229736,
      "step": 1270
    },
    {
      "epoch": 0.6823883592574009,
      "grad_norm": 7.399312373987854,
      "learning_rate": 9.513496170830909e-07,
      "logits/chosen": -0.049418993294239044,
      "logits/rejected": 0.0669780820608139,
      "logps/chosen": -1.4792741537094116,
      "logps/rejected": -1.706899642944336,
      "loss": 0.5554,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4792741537094116,
      "rewards/margins": 0.22762539982795715,
      "rewards/rejected": -1.706899642944336,
      "sft_loss": 1.4689277410507202,
      "step": 1275
    },
    {
      "epoch": 0.6850643920388024,
      "grad_norm": 7.626931519624881,
      "learning_rate": 9.506773217311382e-07,
      "logits/chosen": -0.06780585646629333,
      "logits/rejected": 0.09956692159175873,
      "logps/chosen": -1.4898687601089478,
      "logps/rejected": -1.7180099487304688,
      "loss": 0.5619,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.4898687601089478,
      "rewards/margins": 0.22814103960990906,
      "rewards/rejected": -1.7180099487304688,
      "sft_loss": 1.537276268005371,
      "step": 1280
    },
    {
      "epoch": 0.687740424820204,
      "grad_norm": 5.433214321002391,
      "learning_rate": 9.500006539058334e-07,
      "logits/chosen": -0.02779715694487095,
      "logits/rejected": 0.11424566805362701,
      "logps/chosen": -1.3562198877334595,
      "logps/rejected": -1.6049524545669556,
      "loss": 0.5213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3562198877334595,
      "rewards/margins": 0.2487325221300125,
      "rewards/rejected": -1.6049524545669556,
      "sft_loss": 1.3964496850967407,
      "step": 1285
    },
    {
      "epoch": 0.6904164576016056,
      "grad_norm": 8.29469757854584,
      "learning_rate": 9.493196201722109e-07,
      "logits/chosen": -0.17457318305969238,
      "logits/rejected": -0.01711769960820675,
      "logps/chosen": -1.4438587427139282,
      "logps/rejected": -1.6200523376464844,
      "loss": 0.5669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.4438587427139282,
      "rewards/margins": 0.17619381844997406,
      "rewards/rejected": -1.6200523376464844,
      "sft_loss": 1.4548752307891846,
      "step": 1290
    },
    {
      "epoch": 0.6930924903830072,
      "grad_norm": 5.888598056038247,
      "learning_rate": 9.486342271376628e-07,
      "logits/chosen": -0.08861254900693893,
      "logits/rejected": -0.07075365632772446,
      "logps/chosen": -1.422069787979126,
      "logps/rejected": -1.7725293636322021,
      "loss": 0.5059,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.422069787979126,
      "rewards/margins": 0.3504595160484314,
      "rewards/rejected": -1.7725293636322021,
      "sft_loss": 1.4434196949005127,
      "step": 1295
    },
    {
      "epoch": 0.6957685231644087,
      "grad_norm": 5.2887307915100505,
      "learning_rate": 9.479444814518755e-07,
      "logits/chosen": -0.09308328479528427,
      "logits/rejected": 0.16950467228889465,
      "logps/chosen": -1.3764543533325195,
      "logps/rejected": -1.7472703456878662,
      "loss": 0.5088,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3764543533325195,
      "rewards/margins": 0.3708159327507019,
      "rewards/rejected": -1.7472703456878662,
      "sft_loss": 1.458449125289917,
      "step": 1300
    },
    {
      "epoch": 0.6984445559458103,
      "grad_norm": 5.01071388324762,
      "learning_rate": 9.472503898067645e-07,
      "logits/chosen": 0.007398007903248072,
      "logits/rejected": 0.06001367047429085,
      "logps/chosen": -1.4065968990325928,
      "logps/rejected": -1.6868667602539062,
      "loss": 0.5346,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4065968990325928,
      "rewards/margins": 0.2802698314189911,
      "rewards/rejected": -1.6868667602539062,
      "sft_loss": 1.4229308366775513,
      "step": 1305
    },
    {
      "epoch": 0.701120588727212,
      "grad_norm": 3.931757603333999,
      "learning_rate": 9.465519589364099e-07,
      "logits/chosen": 0.00926266424357891,
      "logits/rejected": 0.09389156103134155,
      "logps/chosen": -1.3826566934585571,
      "logps/rejected": -1.6744199991226196,
      "loss": 0.5238,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3826566934585571,
      "rewards/margins": 0.29176321625709534,
      "rewards/rejected": -1.6744199991226196,
      "sft_loss": 1.4121711254119873,
      "step": 1310
    },
    {
      "epoch": 0.7037966215086134,
      "grad_norm": 6.229548861295008,
      "learning_rate": 9.458491956169914e-07,
      "logits/chosen": -0.0754542127251625,
      "logits/rejected": 0.10710541903972626,
      "logps/chosen": -1.357774019241333,
      "logps/rejected": -1.7196426391601562,
      "loss": 0.4957,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.357774019241333,
      "rewards/margins": 0.36186856031417847,
      "rewards/rejected": -1.7196426391601562,
      "sft_loss": 1.3753013610839844,
      "step": 1315
    },
    {
      "epoch": 0.706472654290015,
      "grad_norm": 4.592870510848051,
      "learning_rate": 9.451421066667215e-07,
      "logits/chosen": -0.18370278179645538,
      "logits/rejected": 0.020677369087934494,
      "logps/chosen": -1.3615787029266357,
      "logps/rejected": -1.6913648843765259,
      "loss": 0.4982,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.3615787029266357,
      "rewards/margins": 0.3297862112522125,
      "rewards/rejected": -1.6913648843765259,
      "sft_loss": 1.3998141288757324,
      "step": 1320
    },
    {
      "epoch": 0.7091486870714167,
      "grad_norm": 7.837754456611585,
      "learning_rate": 9.444306989457805e-07,
      "logits/chosen": -0.03532402962446213,
      "logits/rejected": 0.08233954012393951,
      "logps/chosen": -1.4457212686538696,
      "logps/rejected": -1.7230970859527588,
      "loss": 0.5751,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.4457212686538696,
      "rewards/margins": 0.27737584710121155,
      "rewards/rejected": -1.7230970859527588,
      "sft_loss": 1.4261298179626465,
      "step": 1325
    },
    {
      "epoch": 0.7118247198528181,
      "grad_norm": 6.3131827199709045,
      "learning_rate": 9.437149793562489e-07,
      "logits/chosen": -0.04977645352482796,
      "logits/rejected": 0.0691833645105362,
      "logps/chosen": -1.3893170356750488,
      "logps/rejected": -1.633607268333435,
      "loss": 0.5347,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.3893170356750488,
      "rewards/margins": 0.2442903220653534,
      "rewards/rejected": -1.633607268333435,
      "sft_loss": 1.399949550628662,
      "step": 1330
    },
    {
      "epoch": 0.7145007526342197,
      "grad_norm": 7.695878675767656,
      "learning_rate": 9.429949548420417e-07,
      "logits/chosen": -0.033573225140571594,
      "logits/rejected": 0.046844761818647385,
      "logps/chosen": -1.4621721506118774,
      "logps/rejected": -1.711739182472229,
      "loss": 0.5372,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4621721506118774,
      "rewards/margins": 0.24956703186035156,
      "rewards/rejected": -1.711739182472229,
      "sft_loss": 1.496930480003357,
      "step": 1335
    },
    {
      "epoch": 0.7171767854156214,
      "grad_norm": 6.985545650344274,
      "learning_rate": 9.422706323888396e-07,
      "logits/chosen": -0.01977308839559555,
      "logits/rejected": 0.021173939108848572,
      "logps/chosen": -1.3878123760223389,
      "logps/rejected": -1.6171363592147827,
      "loss": 0.5358,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3878123760223389,
      "rewards/margins": 0.22932401299476624,
      "rewards/rejected": -1.6171363592147827,
      "sft_loss": 1.4066617488861084,
      "step": 1340
    },
    {
      "epoch": 0.719852818197023,
      "grad_norm": 4.366435796291409,
      "learning_rate": 9.415420190240225e-07,
      "logits/chosen": 0.02655363641679287,
      "logits/rejected": 0.2236282378435135,
      "logps/chosen": -1.382165551185608,
      "logps/rejected": -1.6554224491119385,
      "loss": 0.5134,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.382165551185608,
      "rewards/margins": 0.2732568681240082,
      "rewards/rejected": -1.6554224491119385,
      "sft_loss": 1.42926025390625,
      "step": 1345
    },
    {
      "epoch": 0.7225288509784245,
      "grad_norm": 7.81159246638897,
      "learning_rate": 9.408091218166002e-07,
      "logits/chosen": 0.03293774276971817,
      "logits/rejected": 0.09691715985536575,
      "logps/chosen": -1.4025884866714478,
      "logps/rejected": -1.5003225803375244,
      "loss": 0.5684,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4025884866714478,
      "rewards/margins": 0.09773415327072144,
      "rewards/rejected": -1.5003225803375244,
      "sft_loss": 1.440049409866333,
      "step": 1350
    },
    {
      "epoch": 0.7252048837598261,
      "grad_norm": 5.268029857993076,
      "learning_rate": 9.400719478771449e-07,
      "logits/chosen": 0.0003868401108775288,
      "logits/rejected": 0.30421125888824463,
      "logps/chosen": -1.4475411176681519,
      "logps/rejected": -1.6507571935653687,
      "loss": 0.558,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4475411176681519,
      "rewards/margins": 0.20321616530418396,
      "rewards/rejected": -1.6507571935653687,
      "sft_loss": 1.478035569190979,
      "step": 1355
    },
    {
      "epoch": 0.7278809165412277,
      "grad_norm": 5.476104551012313,
      "learning_rate": 9.393305043577209e-07,
      "logits/chosen": -0.1295911818742752,
      "logits/rejected": 0.025414815172553062,
      "logps/chosen": -1.4887486696243286,
      "logps/rejected": -1.822188138961792,
      "loss": 0.5262,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4887486696243286,
      "rewards/margins": 0.3334396481513977,
      "rewards/rejected": -1.822188138961792,
      "sft_loss": 1.5442901849746704,
      "step": 1360
    },
    {
      "epoch": 0.7305569493226292,
      "grad_norm": 4.149863738674073,
      "learning_rate": 9.38584798451817e-07,
      "logits/chosen": -0.029890060424804688,
      "logits/rejected": 0.10929396003484726,
      "logps/chosen": -1.413294792175293,
      "logps/rejected": -1.7026281356811523,
      "loss": 0.5145,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.413294792175293,
      "rewards/margins": 0.2893335521221161,
      "rewards/rejected": -1.7026281356811523,
      "sft_loss": 1.4526488780975342,
      "step": 1365
    },
    {
      "epoch": 0.7332329821040308,
      "grad_norm": 21.2802024194351,
      "learning_rate": 9.37834837394275e-07,
      "logits/chosen": -0.018464690074324608,
      "logits/rejected": 0.10323115438222885,
      "logps/chosen": -1.5000975131988525,
      "logps/rejected": -1.958627700805664,
      "loss": 0.4919,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5000975131988525,
      "rewards/margins": 0.45853009819984436,
      "rewards/rejected": -1.958627700805664,
      "sft_loss": 1.5218764543533325,
      "step": 1370
    },
    {
      "epoch": 0.7359090148854324,
      "grad_norm": 3.655166951636573,
      "learning_rate": 9.370806284612203e-07,
      "logits/chosen": -0.04375447332859039,
      "logits/rejected": 0.11701725423336029,
      "logps/chosen": -1.4266334772109985,
      "logps/rejected": -1.8739063739776611,
      "loss": 0.4893,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4266334772109985,
      "rewards/margins": 0.4472728669643402,
      "rewards/rejected": -1.8739063739776611,
      "sft_loss": 1.4907891750335693,
      "step": 1375
    },
    {
      "epoch": 0.738585047666834,
      "grad_norm": 5.149895071058057,
      "learning_rate": 9.363221789699912e-07,
      "logits/chosen": -0.08814994990825653,
      "logits/rejected": 0.051434438675642014,
      "logps/chosen": -1.497077465057373,
      "logps/rejected": -1.6954923868179321,
      "loss": 0.5753,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.497077465057373,
      "rewards/margins": 0.19841498136520386,
      "rewards/rejected": -1.6954923868179321,
      "sft_loss": 1.4636865854263306,
      "step": 1380
    },
    {
      "epoch": 0.7412610804482355,
      "grad_norm": 8.096091593787765,
      "learning_rate": 9.355594962790682e-07,
      "logits/chosen": -0.06694310158491135,
      "logits/rejected": 0.07419678568840027,
      "logps/chosen": -1.371813178062439,
      "logps/rejected": -1.674665093421936,
      "loss": 0.5163,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.371813178062439,
      "rewards/margins": 0.302852064371109,
      "rewards/rejected": -1.674665093421936,
      "sft_loss": 1.4243870973587036,
      "step": 1385
    },
    {
      "epoch": 0.7439371132296371,
      "grad_norm": 7.542392017959355,
      "learning_rate": 9.34792587788002e-07,
      "logits/chosen": 0.03423604369163513,
      "logits/rejected": 0.15960803627967834,
      "logps/chosen": -1.4509718418121338,
      "logps/rejected": -1.7408100366592407,
      "loss": 0.5235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4509718418121338,
      "rewards/margins": 0.28983819484710693,
      "rewards/rejected": -1.7408100366592407,
      "sft_loss": 1.4992636442184448,
      "step": 1390
    },
    {
      "epoch": 0.7466131460110387,
      "grad_norm": 4.278440356940747,
      "learning_rate": 9.34021460937342e-07,
      "logits/chosen": 0.060794007033109665,
      "logits/rejected": 0.15525543689727783,
      "logps/chosen": -1.4032586812973022,
      "logps/rejected": -1.6212393045425415,
      "loss": 0.5167,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4032586812973022,
      "rewards/margins": 0.21798057854175568,
      "rewards/rejected": -1.6212393045425415,
      "sft_loss": 1.426965594291687,
      "step": 1395
    },
    {
      "epoch": 0.7492891787924402,
      "grad_norm": 6.165930757755512,
      "learning_rate": 9.332461232085646e-07,
      "logits/chosen": -0.13838526606559753,
      "logits/rejected": 0.014215810224413872,
      "logps/chosen": -1.528344988822937,
      "logps/rejected": -1.7559783458709717,
      "loss": 0.5536,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.528344988822937,
      "rewards/margins": 0.22763332724571228,
      "rewards/rejected": -1.7559783458709717,
      "sft_loss": 1.5378443002700806,
      "step": 1400
    },
    {
      "epoch": 0.7519652115738418,
      "grad_norm": 4.358325387270058,
      "learning_rate": 9.324665821239998e-07,
      "logits/chosen": -0.03495486080646515,
      "logits/rejected": 0.15800495445728302,
      "logps/chosen": -1.3264365196228027,
      "logps/rejected": -1.7632360458374023,
      "loss": 0.4909,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3264365196228027,
      "rewards/margins": 0.43679970502853394,
      "rewards/rejected": -1.7632360458374023,
      "sft_loss": 1.4041543006896973,
      "step": 1405
    },
    {
      "epoch": 0.7546412443552434,
      "grad_norm": 8.185094426743838,
      "learning_rate": 9.316828452467583e-07,
      "logits/chosen": -0.06456667929887772,
      "logits/rejected": 0.1312786191701889,
      "logps/chosen": -1.472119688987732,
      "logps/rejected": -1.795741319656372,
      "loss": 0.5151,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.472119688987732,
      "rewards/margins": 0.3236214816570282,
      "rewards/rejected": -1.795741319656372,
      "sft_loss": 1.535501480102539,
      "step": 1410
    },
    {
      "epoch": 0.7573172771366449,
      "grad_norm": 7.66034027661226,
      "learning_rate": 9.30894920180659e-07,
      "logits/chosen": 0.0350164994597435,
      "logits/rejected": 0.19542302191257477,
      "logps/chosen": -1.4765806198120117,
      "logps/rejected": -1.6078450679779053,
      "loss": 0.5647,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4765806198120117,
      "rewards/margins": 0.1312645971775055,
      "rewards/rejected": -1.6078450679779053,
      "sft_loss": 1.4553964138031006,
      "step": 1415
    },
    {
      "epoch": 0.7599933099180465,
      "grad_norm": 5.423041434028237,
      "learning_rate": 9.301028145701543e-07,
      "logits/chosen": 0.04505506902933121,
      "logits/rejected": 0.1933830976486206,
      "logps/chosen": -1.3929471969604492,
      "logps/rejected": -1.8875548839569092,
      "loss": 0.4984,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3929471969604492,
      "rewards/margins": 0.4946078360080719,
      "rewards/rejected": -1.8875548839569092,
      "sft_loss": 1.4555342197418213,
      "step": 1420
    },
    {
      "epoch": 0.7626693426994481,
      "grad_norm": 5.903348939683053,
      "learning_rate": 9.293065361002563e-07,
      "logits/chosen": 0.05219608545303345,
      "logits/rejected": 0.15234455466270447,
      "logps/chosen": -1.4215357303619385,
      "logps/rejected": -1.8510538339614868,
      "loss": 0.4954,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.4215357303619385,
      "rewards/margins": 0.4295181334018707,
      "rewards/rejected": -1.8510538339614868,
      "sft_loss": 1.4525610208511353,
      "step": 1425
    },
    {
      "epoch": 0.7653453754808497,
      "grad_norm": 8.799420024776662,
      "learning_rate": 9.285060924964622e-07,
      "logits/chosen": -0.045768219977617264,
      "logits/rejected": 0.10385224968194962,
      "logps/chosen": -1.5010254383087158,
      "logps/rejected": -1.7813167572021484,
      "loss": 0.5277,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5010254383087158,
      "rewards/margins": 0.2802914083003998,
      "rewards/rejected": -1.7813167572021484,
      "sft_loss": 1.4974098205566406,
      "step": 1430
    },
    {
      "epoch": 0.7680214082622512,
      "grad_norm": 4.098418469896391,
      "learning_rate": 9.277014915246792e-07,
      "logits/chosen": 0.11667392402887344,
      "logits/rejected": 0.1806088089942932,
      "logps/chosen": -1.3921082019805908,
      "logps/rejected": -1.8272292613983154,
      "loss": 0.4879,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3921082019805908,
      "rewards/margins": 0.4351211488246918,
      "rewards/rejected": -1.8272292613983154,
      "sft_loss": 1.4282985925674438,
      "step": 1435
    },
    {
      "epoch": 0.7706974410436528,
      "grad_norm": 4.37448697414661,
      "learning_rate": 9.268927409911498e-07,
      "logits/chosen": 0.0001061245784512721,
      "logits/rejected": 0.1211082935333252,
      "logps/chosen": -1.4377018213272095,
      "logps/rejected": -1.6614086627960205,
      "loss": 0.5484,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4377018213272095,
      "rewards/margins": 0.2237069308757782,
      "rewards/rejected": -1.6614086627960205,
      "sft_loss": 1.5097572803497314,
      "step": 1440
    },
    {
      "epoch": 0.7733734738250544,
      "grad_norm": 7.801656820601944,
      "learning_rate": 9.260798487423749e-07,
      "logits/chosen": -0.0659121721982956,
      "logits/rejected": 0.16661436855793,
      "logps/chosen": -1.4803264141082764,
      "logps/rejected": -1.7045660018920898,
      "loss": 0.5617,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4803264141082764,
      "rewards/margins": 0.22423963248729706,
      "rewards/rejected": -1.7045660018920898,
      "sft_loss": 1.5604345798492432,
      "step": 1445
    },
    {
      "epoch": 0.7760495066064559,
      "grad_norm": 9.88040393877702,
      "learning_rate": 9.252628226650389e-07,
      "logits/chosen": 0.060816846787929535,
      "logits/rejected": 0.16335485875606537,
      "logps/chosen": -1.4167139530181885,
      "logps/rejected": -1.654566764831543,
      "loss": 0.5477,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4167139530181885,
      "rewards/margins": 0.23785285651683807,
      "rewards/rejected": -1.654566764831543,
      "sft_loss": 1.4502527713775635,
      "step": 1450
    },
    {
      "epoch": 0.7787255393878575,
      "grad_norm": 6.799381513616858,
      "learning_rate": 9.244416706859321e-07,
      "logits/chosen": -0.011297956109046936,
      "logits/rejected": 0.16043448448181152,
      "logps/chosen": -1.3892749547958374,
      "logps/rejected": -1.7083572149276733,
      "loss": 0.5188,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3892749547958374,
      "rewards/margins": 0.31908220052719116,
      "rewards/rejected": -1.7083572149276733,
      "sft_loss": 1.4365092515945435,
      "step": 1455
    },
    {
      "epoch": 0.7814015721692591,
      "grad_norm": 4.425286877544062,
      "learning_rate": 9.23616400771875e-07,
      "logits/chosen": 0.01945159211754799,
      "logits/rejected": 0.20056121051311493,
      "logps/chosen": -1.3618758916854858,
      "logps/rejected": -1.7152411937713623,
      "loss": 0.4964,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3618758916854858,
      "rewards/margins": 0.35336512327194214,
      "rewards/rejected": -1.7152411937713623,
      "sft_loss": 1.3751140832901,
      "step": 1460
    },
    {
      "epoch": 0.7840776049506607,
      "grad_norm": 7.25832869318277,
      "learning_rate": 9.227870209296395e-07,
      "logits/chosen": 0.060930054634809494,
      "logits/rejected": 0.18895120918750763,
      "logps/chosen": -1.4787466526031494,
      "logps/rejected": -1.742435097694397,
      "loss": 0.548,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4787466526031494,
      "rewards/margins": 0.26368841528892517,
      "rewards/rejected": -1.742435097694397,
      "sft_loss": 1.5360684394836426,
      "step": 1465
    },
    {
      "epoch": 0.7867536377320622,
      "grad_norm": 4.933151756958093,
      "learning_rate": 9.219535392058728e-07,
      "logits/chosen": -0.023029694333672523,
      "logits/rejected": 0.009718982502818108,
      "logps/chosen": -1.4260733127593994,
      "logps/rejected": -1.7250900268554688,
      "loss": 0.5289,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.4260733127593994,
      "rewards/margins": 0.29901689291000366,
      "rewards/rejected": -1.7250900268554688,
      "sft_loss": 1.4916422367095947,
      "step": 1470
    },
    {
      "epoch": 0.7894296705134638,
      "grad_norm": 6.560511701032652,
      "learning_rate": 9.211159636870181e-07,
      "logits/chosen": -0.0503508560359478,
      "logits/rejected": 0.13822703063488007,
      "logps/chosen": -1.4035285711288452,
      "logps/rejected": -1.7277904748916626,
      "loss": 0.5121,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4035285711288452,
      "rewards/margins": 0.3242620825767517,
      "rewards/rejected": -1.7277904748916626,
      "sft_loss": 1.4790805578231812,
      "step": 1475
    },
    {
      "epoch": 0.7921057032948654,
      "grad_norm": 4.958156966185051,
      "learning_rate": 9.202743024992367e-07,
      "logits/chosen": 0.051444463431835175,
      "logits/rejected": 0.17066633701324463,
      "logps/chosen": -1.3592405319213867,
      "logps/rejected": -1.7749545574188232,
      "loss": 0.4929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3592405319213867,
      "rewards/margins": 0.4157140254974365,
      "rewards/rejected": -1.7749545574188232,
      "sft_loss": 1.4301153421401978,
      "step": 1480
    },
    {
      "epoch": 0.7947817360762669,
      "grad_norm": 5.009038213095197,
      "learning_rate": 9.194285638083293e-07,
      "logits/chosen": 0.07895182073116302,
      "logits/rejected": 0.2562471032142639,
      "logps/chosen": -1.4837896823883057,
      "logps/rejected": -1.892653226852417,
      "loss": 0.5159,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4837896823883057,
      "rewards/margins": 0.4088636338710785,
      "rewards/rejected": -1.892653226852417,
      "sft_loss": 1.4823133945465088,
      "step": 1485
    },
    {
      "epoch": 0.7974577688576685,
      "grad_norm": 6.097392272653846,
      "learning_rate": 9.185787558196562e-07,
      "logits/chosen": 0.00459608668461442,
      "logits/rejected": 0.13557849824428558,
      "logps/chosen": -1.4189090728759766,
      "logps/rejected": -1.684511423110962,
      "loss": 0.5499,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4189090728759766,
      "rewards/margins": 0.2656022608280182,
      "rewards/rejected": -1.684511423110962,
      "sft_loss": 1.4373775720596313,
      "step": 1490
    },
    {
      "epoch": 0.8001338016390701,
      "grad_norm": 9.718433536349252,
      "learning_rate": 9.177248867780583e-07,
      "logits/chosen": 0.003417456056922674,
      "logits/rejected": 0.1183265894651413,
      "logps/chosen": -1.586362600326538,
      "logps/rejected": -1.7566999197006226,
      "loss": 0.5891,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.586362600326538,
      "rewards/margins": 0.1703372746706009,
      "rewards/rejected": -1.7566999197006226,
      "sft_loss": 1.6519889831542969,
      "step": 1495
    },
    {
      "epoch": 0.8028098344204716,
      "grad_norm": 10.362820259678747,
      "learning_rate": 9.168669649677769e-07,
      "logits/chosen": -0.0650627538561821,
      "logits/rejected": 0.04227939993143082,
      "logps/chosen": -1.4759619235992432,
      "logps/rejected": -1.8760197162628174,
      "loss": 0.5195,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.4759619235992432,
      "rewards/margins": 0.40005773305892944,
      "rewards/rejected": -1.8760197162628174,
      "sft_loss": 1.5462987422943115,
      "step": 1500
    },
    {
      "epoch": 0.8054858672018732,
      "grad_norm": 7.091050987105003,
      "learning_rate": 9.16004998712373e-07,
      "logits/chosen": 0.05563043802976608,
      "logits/rejected": 0.13614422082901,
      "logps/chosen": -1.4341747760772705,
      "logps/rejected": -1.8504860401153564,
      "loss": 0.4909,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4341747760772705,
      "rewards/margins": 0.416311115026474,
      "rewards/rejected": -1.8504860401153564,
      "sft_loss": 1.4413312673568726,
      "step": 1505
    },
    {
      "epoch": 0.8081618999832748,
      "grad_norm": 3.8930626725283717,
      "learning_rate": 9.151389963746472e-07,
      "logits/chosen": -0.03380414843559265,
      "logits/rejected": 0.27143990993499756,
      "logps/chosen": -1.4646518230438232,
      "logps/rejected": -1.8340686559677124,
      "loss": 0.5071,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4646518230438232,
      "rewards/margins": 0.36941686272621155,
      "rewards/rejected": -1.8340686559677124,
      "sft_loss": 1.4922515153884888,
      "step": 1510
    },
    {
      "epoch": 0.8108379327646764,
      "grad_norm": 5.530315878153944,
      "learning_rate": 9.142689663565577e-07,
      "logits/chosen": 0.06472794711589813,
      "logits/rejected": 0.1404540240764618,
      "logps/chosen": -1.4131653308868408,
      "logps/rejected": -1.723859429359436,
      "loss": 0.5189,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4131653308868408,
      "rewards/margins": 0.3106943964958191,
      "rewards/rejected": -1.723859429359436,
      "sft_loss": 1.4428884983062744,
      "step": 1515
    },
    {
      "epoch": 0.8135139655460779,
      "grad_norm": 6.649544973120154,
      "learning_rate": 9.133949170991397e-07,
      "logits/chosen": 0.02315221168100834,
      "logits/rejected": 0.11982355266809464,
      "logps/chosen": -1.4375369548797607,
      "logps/rejected": -1.7398595809936523,
      "loss": 0.5129,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4375369548797607,
      "rewards/margins": 0.3023225963115692,
      "rewards/rejected": -1.7398595809936523,
      "sft_loss": 1.5597889423370361,
      "step": 1520
    },
    {
      "epoch": 0.8161899983274795,
      "grad_norm": 4.798262164071743,
      "learning_rate": 9.125168570824231e-07,
      "logits/chosen": -0.014184604398906231,
      "logits/rejected": 0.18807581067085266,
      "logps/chosen": -1.4288346767425537,
      "logps/rejected": -1.6621601581573486,
      "loss": 0.5507,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4288346767425537,
      "rewards/margins": 0.23332540690898895,
      "rewards/rejected": -1.6621601581573486,
      "sft_loss": 1.4449572563171387,
      "step": 1525
    },
    {
      "epoch": 0.8188660311088811,
      "grad_norm": 7.721218526827049,
      "learning_rate": 9.116347948253496e-07,
      "logits/chosen": -0.05206465721130371,
      "logits/rejected": 0.06865311414003372,
      "logps/chosen": -1.4678184986114502,
      "logps/rejected": -1.7325515747070312,
      "loss": 0.5415,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.4678184986114502,
      "rewards/margins": 0.2647330164909363,
      "rewards/rejected": -1.7325515747070312,
      "sft_loss": 1.4634746313095093,
      "step": 1530
    },
    {
      "epoch": 0.8215420638902826,
      "grad_norm": 7.566815023460756,
      "learning_rate": 9.107487388856916e-07,
      "logits/chosen": -0.04291130229830742,
      "logits/rejected": 0.14794987440109253,
      "logps/chosen": -1.3930656909942627,
      "logps/rejected": -1.6991512775421143,
      "loss": 0.4948,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.3930656909942627,
      "rewards/margins": 0.3060855269432068,
      "rewards/rejected": -1.6991512775421143,
      "sft_loss": 1.4516359567642212,
      "step": 1535
    },
    {
      "epoch": 0.8242180966716842,
      "grad_norm": 8.855577325246788,
      "learning_rate": 9.098586978599673e-07,
      "logits/chosen": 0.03489986062049866,
      "logits/rejected": 0.21353301405906677,
      "logps/chosen": -1.4300438165664673,
      "logps/rejected": -1.8971660137176514,
      "loss": 0.4705,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4300438165664673,
      "rewards/margins": 0.46712246537208557,
      "rewards/rejected": -1.8971660137176514,
      "sft_loss": 1.4638926982879639,
      "step": 1540
    },
    {
      "epoch": 0.8268941294530858,
      "grad_norm": 5.968612534315002,
      "learning_rate": 9.089646803833588e-07,
      "logits/chosen": 0.04957198724150658,
      "logits/rejected": 0.23397859930992126,
      "logps/chosen": -1.448665738105774,
      "logps/rejected": -1.7475725412368774,
      "loss": 0.5269,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.448665738105774,
      "rewards/margins": 0.29890674352645874,
      "rewards/rejected": -1.7475725412368774,
      "sft_loss": 1.497968077659607,
      "step": 1545
    },
    {
      "epoch": 0.8295701622344873,
      "grad_norm": 5.774737608548478,
      "learning_rate": 9.080666951296276e-07,
      "logits/chosen": -0.10732851177453995,
      "logits/rejected": 0.19018636643886566,
      "logps/chosen": -1.4928748607635498,
      "logps/rejected": -1.8812583684921265,
      "loss": 0.5216,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4928748607635498,
      "rewards/margins": 0.3883834481239319,
      "rewards/rejected": -1.8812583684921265,
      "sft_loss": 1.5361144542694092,
      "step": 1550
    },
    {
      "epoch": 0.8322461950158889,
      "grad_norm": 5.013952174103058,
      "learning_rate": 9.071647508110305e-07,
      "logits/chosen": -0.0828329548239708,
      "logits/rejected": 0.19987811148166656,
      "logps/chosen": -1.5106512308120728,
      "logps/rejected": -1.9506137371063232,
      "loss": 0.5353,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5106512308120728,
      "rewards/margins": 0.4399626851081848,
      "rewards/rejected": -1.9506137371063232,
      "sft_loss": 1.4890204668045044,
      "step": 1555
    },
    {
      "epoch": 0.8349222277972905,
      "grad_norm": 7.321461068617166,
      "learning_rate": 9.062588561782354e-07,
      "logits/chosen": 0.04645683243870735,
      "logits/rejected": 0.1263210028409958,
      "logps/chosen": -1.563812017440796,
      "logps/rejected": -1.8573408126831055,
      "loss": 0.5612,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.563812017440796,
      "rewards/margins": 0.2935287654399872,
      "rewards/rejected": -1.8573408126831055,
      "sft_loss": 1.6140215396881104,
      "step": 1560
    },
    {
      "epoch": 0.8375982605786921,
      "grad_norm": 4.6835886280890735,
      "learning_rate": 9.053490200202358e-07,
      "logits/chosen": 0.018405133858323097,
      "logits/rejected": 0.11775252968072891,
      "logps/chosen": -1.52276611328125,
      "logps/rejected": -1.820581078529358,
      "loss": 0.5372,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.52276611328125,
      "rewards/margins": 0.29781508445739746,
      "rewards/rejected": -1.820581078529358,
      "sft_loss": 1.5765827894210815,
      "step": 1565
    },
    {
      "epoch": 0.8402742933600936,
      "grad_norm": 13.799267959660574,
      "learning_rate": 9.044352511642661e-07,
      "logits/chosen": 0.054694343358278275,
      "logits/rejected": 0.08119392395019531,
      "logps/chosen": -1.425003170967102,
      "logps/rejected": -1.666555643081665,
      "loss": 0.5489,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.425003170967102,
      "rewards/margins": 0.24155232310295105,
      "rewards/rejected": -1.666555643081665,
      "sft_loss": 1.4962987899780273,
      "step": 1570
    },
    {
      "epoch": 0.8429503261414952,
      "grad_norm": 5.962658386421319,
      "learning_rate": 9.03517558475716e-07,
      "logits/chosen": 0.006678787060081959,
      "logits/rejected": 0.12651744484901428,
      "logps/chosen": -1.430008053779602,
      "logps/rejected": -1.665032148361206,
      "loss": 0.5369,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.430008053779602,
      "rewards/margins": 0.23502404987812042,
      "rewards/rejected": -1.665032148361206,
      "sft_loss": 1.466813325881958,
      "step": 1575
    },
    {
      "epoch": 0.8456263589228968,
      "grad_norm": 7.66418865064177,
      "learning_rate": 9.025959508580436e-07,
      "logits/chosen": 0.0546504482626915,
      "logits/rejected": 0.3250477910041809,
      "logps/chosen": -1.4601762294769287,
      "logps/rejected": -1.8048124313354492,
      "loss": 0.5084,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4601762294769287,
      "rewards/margins": 0.34463605284690857,
      "rewards/rejected": -1.8048124313354492,
      "sft_loss": 1.496201992034912,
      "step": 1580
    },
    {
      "epoch": 0.8483023917042983,
      "grad_norm": 4.515357989887072,
      "learning_rate": 9.016704372526905e-07,
      "logits/chosen": 0.0008736684685572982,
      "logits/rejected": 0.1726742684841156,
      "logps/chosen": -1.3656678199768066,
      "logps/rejected": -1.7986243963241577,
      "loss": 0.4765,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3656678199768066,
      "rewards/margins": 0.4329564571380615,
      "rewards/rejected": -1.7986243963241577,
      "sft_loss": 1.3878506422042847,
      "step": 1585
    },
    {
      "epoch": 0.8509784244856999,
      "grad_norm": 10.693019607988258,
      "learning_rate": 9.007410266389934e-07,
      "logits/chosen": -0.07369935512542725,
      "logits/rejected": 0.027142727747559547,
      "logps/chosen": -1.3922083377838135,
      "logps/rejected": -1.6310374736785889,
      "loss": 0.5281,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3922083377838135,
      "rewards/margins": 0.23882922530174255,
      "rewards/rejected": -1.6310374736785889,
      "sft_loss": 1.4906740188598633,
      "step": 1590
    },
    {
      "epoch": 0.8536544572671015,
      "grad_norm": 8.812585236392314,
      "learning_rate": 8.998077280340981e-07,
      "logits/chosen": 0.014504333958029747,
      "logits/rejected": 0.1055087074637413,
      "logps/chosen": -1.5411064624786377,
      "logps/rejected": -1.707898497581482,
      "loss": 0.5896,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.5411064624786377,
      "rewards/margins": 0.16679205000400543,
      "rewards/rejected": -1.707898497581482,
      "sft_loss": 1.5593599081039429,
      "step": 1595
    },
    {
      "epoch": 0.8563304900485031,
      "grad_norm": 3.5492870174689735,
      "learning_rate": 8.988705504928722e-07,
      "logits/chosen": -0.10841546207666397,
      "logits/rejected": 0.10659325122833252,
      "logps/chosen": -1.4652018547058105,
      "logps/rejected": -1.883522391319275,
      "loss": 0.4953,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4652018547058105,
      "rewards/margins": 0.41832059621810913,
      "rewards/rejected": -1.883522391319275,
      "sft_loss": 1.5147536993026733,
      "step": 1600
    },
    {
      "epoch": 0.8563304900485031,
      "eval_logits/chosen": 0.27520132064819336,
      "eval_logits/rejected": 0.37425851821899414,
      "eval_logps/chosen": -1.4448524713516235,
      "eval_logps/rejected": -1.7957645654678345,
      "eval_loss": 0.5101240873336792,
      "eval_rewards/accuracies": 0.6261127591133118,
      "eval_rewards/chosen": -1.4448524713516235,
      "eval_rewards/margins": 0.3509122133255005,
      "eval_rewards/rejected": -1.7957645654678345,
      "eval_runtime": 43.2512,
      "eval_samples_per_second": 31.097,
      "eval_sft_loss": 1.497976541519165,
      "eval_steps_per_second": 7.792,
      "step": 1600
    },
    {
      "epoch": 0.8590065228299046,
      "grad_norm": 6.1013220165784485,
      "learning_rate": 8.979295031078157e-07,
      "logits/chosen": -0.09389957785606384,
      "logits/rejected": 0.15035173296928406,
      "logps/chosen": -1.4221055507659912,
      "logps/rejected": -1.7961909770965576,
      "loss": 0.5,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4221055507659912,
      "rewards/margins": 0.3740856349468231,
      "rewards/rejected": -1.7961909770965576,
      "sft_loss": 1.487693428993225,
      "step": 1605
    },
    {
      "epoch": 0.8616825556113062,
      "grad_norm": 4.059476077333794,
      "learning_rate": 8.969845950089751e-07,
      "logits/chosen": -0.09287907928228378,
      "logits/rejected": 0.10007772594690323,
      "logps/chosen": -1.4006273746490479,
      "logps/rejected": -1.8340444564819336,
      "loss": 0.4891,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4006273746490479,
      "rewards/margins": 0.43341708183288574,
      "rewards/rejected": -1.8340444564819336,
      "sft_loss": 1.4587966203689575,
      "step": 1610
    },
    {
      "epoch": 0.8643585883927078,
      "grad_norm": 7.136162921385605,
      "learning_rate": 8.960358353638526e-07,
      "logits/chosen": -0.05847252532839775,
      "logits/rejected": 0.06235013157129288,
      "logps/chosen": -1.4863847494125366,
      "logps/rejected": -1.8875106573104858,
      "loss": 0.5281,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4863847494125366,
      "rewards/margins": 0.401125967502594,
      "rewards/rejected": -1.8875106573104858,
      "sft_loss": 1.5500595569610596,
      "step": 1615
    },
    {
      "epoch": 0.8670346211741093,
      "grad_norm": 5.857959777310137,
      "learning_rate": 8.950832333773184e-07,
      "logits/chosen": -0.028884366154670715,
      "logits/rejected": 0.1254526674747467,
      "logps/chosen": -1.370598316192627,
      "logps/rejected": -1.7824159860610962,
      "loss": 0.4933,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.370598316192627,
      "rewards/margins": 0.4118177890777588,
      "rewards/rejected": -1.7824159860610962,
      "sft_loss": 1.4120441675186157,
      "step": 1620
    },
    {
      "epoch": 0.869710653955511,
      "grad_norm": 5.859143629683388,
      "learning_rate": 8.941267982915213e-07,
      "logits/chosen": 0.06796115636825562,
      "logits/rejected": 0.11744487285614014,
      "logps/chosen": -1.5567047595977783,
      "logps/rejected": -1.7756681442260742,
      "loss": 0.5867,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5567047595977783,
      "rewards/margins": 0.21896354854106903,
      "rewards/rejected": -1.7756681442260742,
      "sft_loss": 1.5273792743682861,
      "step": 1625
    },
    {
      "epoch": 0.8723866867369126,
      "grad_norm": 5.929078630089166,
      "learning_rate": 8.931665393857983e-07,
      "logits/chosen": -0.02454587072134018,
      "logits/rejected": 0.1381051242351532,
      "logps/chosen": -1.4669086933135986,
      "logps/rejected": -1.8011709451675415,
      "loss": 0.528,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4669086933135986,
      "rewards/margins": 0.33426228165626526,
      "rewards/rejected": -1.8011709451675415,
      "sft_loss": 1.487455129623413,
      "step": 1630
    },
    {
      "epoch": 0.875062719518314,
      "grad_norm": 5.423182196882485,
      "learning_rate": 8.922024659765861e-07,
      "logits/chosen": -0.1283598393201828,
      "logits/rejected": 0.0018849506741389632,
      "logps/chosen": -1.3589659929275513,
      "logps/rejected": -1.7768265008926392,
      "loss": 0.489,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3589659929275513,
      "rewards/margins": 0.4178605079650879,
      "rewards/rejected": -1.7768265008926392,
      "sft_loss": 1.4241197109222412,
      "step": 1635
    },
    {
      "epoch": 0.8777387522997157,
      "grad_norm": 5.874207795306028,
      "learning_rate": 8.912345874173288e-07,
      "logits/chosen": -0.09055529534816742,
      "logits/rejected": 0.0466456413269043,
      "logps/chosen": -1.4008266925811768,
      "logps/rejected": -1.8529808521270752,
      "loss": 0.49,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4008266925811768,
      "rewards/margins": 0.4521542489528656,
      "rewards/rejected": -1.8529808521270752,
      "sft_loss": 1.4470208883285522,
      "step": 1640
    },
    {
      "epoch": 0.8804147850811173,
      "grad_norm": 5.7827022662756065,
      "learning_rate": 8.902629130983885e-07,
      "logits/chosen": -0.008395682089030743,
      "logits/rejected": 0.056283582001924515,
      "logps/chosen": -1.3998442888259888,
      "logps/rejected": -1.6840111017227173,
      "loss": 0.5135,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3998442888259888,
      "rewards/margins": 0.2841669023036957,
      "rewards/rejected": -1.6840111017227173,
      "sft_loss": 1.4410978555679321,
      "step": 1645
    },
    {
      "epoch": 0.8830908178625189,
      "grad_norm": 7.954396025512038,
      "learning_rate": 8.892874524469537e-07,
      "logits/chosen": 0.049496881663799286,
      "logits/rejected": 0.12194067239761353,
      "logps/chosen": -1.410659670829773,
      "logps/rejected": -1.7615478038787842,
      "loss": 0.4846,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.410659670829773,
      "rewards/margins": 0.35088807344436646,
      "rewards/rejected": -1.7615478038787842,
      "sft_loss": 1.4320547580718994,
      "step": 1650
    },
    {
      "epoch": 0.8857668506439204,
      "grad_norm": 5.994738412694896,
      "learning_rate": 8.883082149269478e-07,
      "logits/chosen": -0.06979848444461823,
      "logits/rejected": 0.055761612951755524,
      "logps/chosen": -1.4576666355133057,
      "logps/rejected": -1.745171308517456,
      "loss": 0.5301,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4576666355133057,
      "rewards/margins": 0.2875046133995056,
      "rewards/rejected": -1.745171308517456,
      "sft_loss": 1.4637020826339722,
      "step": 1655
    },
    {
      "epoch": 0.888442883425322,
      "grad_norm": 5.199844719026512,
      "learning_rate": 8.873252100389377e-07,
      "logits/chosen": 0.007501667831093073,
      "logits/rejected": 0.036503929644823074,
      "logps/chosen": -1.3404067754745483,
      "logps/rejected": -1.6517269611358643,
      "loss": 0.5083,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3404067754745483,
      "rewards/margins": 0.3113202154636383,
      "rewards/rejected": -1.6517269611358643,
      "sft_loss": 1.310537338256836,
      "step": 1660
    },
    {
      "epoch": 0.8911189162067236,
      "grad_norm": 3.984860240033483,
      "learning_rate": 8.863384473200411e-07,
      "logits/chosen": -0.025994906201958656,
      "logits/rejected": 0.048259999603033066,
      "logps/chosen": -1.4862369298934937,
      "logps/rejected": -1.7005221843719482,
      "loss": 0.5652,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4862369298934937,
      "rewards/margins": 0.21428528428077698,
      "rewards/rejected": -1.7005221843719482,
      "sft_loss": 1.4881370067596436,
      "step": 1665
    },
    {
      "epoch": 0.8937949489881251,
      "grad_norm": 5.232632139565415,
      "learning_rate": 8.853479363438342e-07,
      "logits/chosen": 0.02264488860964775,
      "logits/rejected": 0.20778794586658478,
      "logps/chosen": -1.504136323928833,
      "logps/rejected": -1.7099090814590454,
      "loss": 0.5853,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.504136323928833,
      "rewards/margins": 0.20577266812324524,
      "rewards/rejected": -1.7099090814590454,
      "sft_loss": 1.464942216873169,
      "step": 1670
    },
    {
      "epoch": 0.8964709817695267,
      "grad_norm": 5.495620916985516,
      "learning_rate": 8.843536867202588e-07,
      "logits/chosen": 0.028523176908493042,
      "logits/rejected": 0.25938358902931213,
      "logps/chosen": -1.4731388092041016,
      "logps/rejected": -1.8747011423110962,
      "loss": 0.5124,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4731388092041016,
      "rewards/margins": 0.40156230330467224,
      "rewards/rejected": -1.8747011423110962,
      "sft_loss": 1.527525544166565,
      "step": 1675
    },
    {
      "epoch": 0.8991470145509283,
      "grad_norm": 6.408484159485986,
      "learning_rate": 8.833557080955292e-07,
      "logits/chosen": -0.07703231275081635,
      "logits/rejected": 0.050923220813274384,
      "logps/chosen": -1.4904206991195679,
      "logps/rejected": -1.7379696369171143,
      "loss": 0.5397,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4904206991195679,
      "rewards/margins": 0.24754881858825684,
      "rewards/rejected": -1.7379696369171143,
      "sft_loss": 1.5126824378967285,
      "step": 1680
    },
    {
      "epoch": 0.9018230473323299,
      "grad_norm": 6.594947547527362,
      "learning_rate": 8.823540101520381e-07,
      "logits/chosen": -0.0388483852148056,
      "logits/rejected": 0.2138877660036087,
      "logps/chosen": -1.4265109300613403,
      "logps/rejected": -1.7498865127563477,
      "loss": 0.5476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4265109300613403,
      "rewards/margins": 0.32337552309036255,
      "rewards/rejected": -1.7498865127563477,
      "sft_loss": 1.4754232168197632,
      "step": 1685
    },
    {
      "epoch": 0.9044990801137314,
      "grad_norm": 5.115040645172101,
      "learning_rate": 8.813486026082637e-07,
      "logits/chosen": -0.035510290414094925,
      "logits/rejected": 0.17635096609592438,
      "logps/chosen": -1.3613451719284058,
      "logps/rejected": -1.716199517250061,
      "loss": 0.4803,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3613451719284058,
      "rewards/margins": 0.3548543155193329,
      "rewards/rejected": -1.716199517250061,
      "sft_loss": 1.4790536165237427,
      "step": 1690
    },
    {
      "epoch": 0.907175112895133,
      "grad_norm": 8.074996039783558,
      "learning_rate": 8.803394952186742e-07,
      "logits/chosen": -0.16678813099861145,
      "logits/rejected": 0.001837402582168579,
      "logps/chosen": -1.4480372667312622,
      "logps/rejected": -1.7357591390609741,
      "loss": 0.5316,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4480372667312622,
      "rewards/margins": 0.2877218723297119,
      "rewards/rejected": -1.7357591390609741,
      "sft_loss": 1.5501583814620972,
      "step": 1695
    },
    {
      "epoch": 0.9098511456765346,
      "grad_norm": 6.995634014615492,
      "learning_rate": 8.793266977736342e-07,
      "logits/chosen": 0.01545445155352354,
      "logits/rejected": -0.03494206815958023,
      "logps/chosen": -1.494322419166565,
      "logps/rejected": -1.631588339805603,
      "loss": 0.5741,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.494322419166565,
      "rewards/margins": 0.13726598024368286,
      "rewards/rejected": -1.631588339805603,
      "sft_loss": 1.5300920009613037,
      "step": 1700
    },
    {
      "epoch": 0.9125271784579361,
      "grad_norm": 9.560309320263531,
      "learning_rate": 8.783102200993085e-07,
      "logits/chosen": -0.010472054593265057,
      "logits/rejected": 0.15402260422706604,
      "logps/chosen": -1.4279364347457886,
      "logps/rejected": -1.6980457305908203,
      "loss": 0.5178,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4279364347457886,
      "rewards/margins": 0.2701093554496765,
      "rewards/rejected": -1.6980457305908203,
      "sft_loss": 1.4880188703536987,
      "step": 1705
    },
    {
      "epoch": 0.9152032112393377,
      "grad_norm": 5.428743027890913,
      "learning_rate": 8.772900720575683e-07,
      "logits/chosen": 0.0033090352080762386,
      "logits/rejected": 0.10284850746393204,
      "logps/chosen": -1.4249826669692993,
      "logps/rejected": -1.7031883001327515,
      "loss": 0.5176,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.4249826669692993,
      "rewards/margins": 0.27820563316345215,
      "rewards/rejected": -1.7031883001327515,
      "sft_loss": 1.4983646869659424,
      "step": 1710
    },
    {
      "epoch": 0.9178792440207393,
      "grad_norm": 5.7398379298109345,
      "learning_rate": 8.762662635458944e-07,
      "logits/chosen": -0.02407381683588028,
      "logits/rejected": 0.2022133320569992,
      "logps/chosen": -1.5204074382781982,
      "logps/rejected": -1.7571582794189453,
      "loss": 0.5779,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.5204074382781982,
      "rewards/margins": 0.23675091564655304,
      "rewards/rejected": -1.7571582794189453,
      "sft_loss": 1.5273138284683228,
      "step": 1715
    },
    {
      "epoch": 0.9205552768021408,
      "grad_norm": 5.898544013989893,
      "learning_rate": 8.752388044972811e-07,
      "logits/chosen": -0.024211686104536057,
      "logits/rejected": 0.05473152548074722,
      "logps/chosen": -1.3195667266845703,
      "logps/rejected": -1.7765014171600342,
      "loss": 0.47,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3195667266845703,
      "rewards/margins": 0.45693454146385193,
      "rewards/rejected": -1.7765014171600342,
      "sft_loss": 1.383522868156433,
      "step": 1720
    },
    {
      "epoch": 0.9232313095835424,
      "grad_norm": 4.328819037966786,
      "learning_rate": 8.74207704880141e-07,
      "logits/chosen": -0.010368740186095238,
      "logits/rejected": 0.11343619972467422,
      "logps/chosen": -1.508044958114624,
      "logps/rejected": -1.948987364768982,
      "loss": 0.5062,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.508044958114624,
      "rewards/margins": 0.4409424364566803,
      "rewards/rejected": -1.948987364768982,
      "sft_loss": 1.5886409282684326,
      "step": 1725
    },
    {
      "epoch": 0.925907342364944,
      "grad_norm": 6.320156944153655,
      "learning_rate": 8.731729746982068e-07,
      "logits/chosen": 0.07320670038461685,
      "logits/rejected": 0.13631558418273926,
      "logps/chosen": -1.4563207626342773,
      "logps/rejected": -1.6941626071929932,
      "loss": 0.5495,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.4563207626342773,
      "rewards/margins": 0.2378418892621994,
      "rewards/rejected": -1.6941626071929932,
      "sft_loss": 1.4958994388580322,
      "step": 1730
    },
    {
      "epoch": 0.9285833751463456,
      "grad_norm": 8.404853592021109,
      "learning_rate": 8.721346239904355e-07,
      "logits/chosen": -0.11669330298900604,
      "logits/rejected": 0.07316017150878906,
      "logps/chosen": -1.4231197834014893,
      "logps/rejected": -2.047053813934326,
      "loss": 0.4633,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4231197834014893,
      "rewards/margins": 0.6239344477653503,
      "rewards/rejected": -2.047053813934326,
      "sft_loss": 1.414732575416565,
      "step": 1735
    },
    {
      "epoch": 0.9312594079277471,
      "grad_norm": 8.028394240012739,
      "learning_rate": 8.710926628309101e-07,
      "logits/chosen": -0.0647435188293457,
      "logits/rejected": 0.10035815089941025,
      "logps/chosen": -1.4947988986968994,
      "logps/rejected": -1.8533751964569092,
      "loss": 0.5075,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4947988986968994,
      "rewards/margins": 0.35857629776000977,
      "rewards/rejected": -1.8533751964569092,
      "sft_loss": 1.4600107669830322,
      "step": 1740
    },
    {
      "epoch": 0.9339354407091487,
      "grad_norm": 4.238728475751365,
      "learning_rate": 8.700471013287424e-07,
      "logits/chosen": 0.022504812106490135,
      "logits/rejected": 0.06147592142224312,
      "logps/chosen": -1.4899694919586182,
      "logps/rejected": -1.8011322021484375,
      "loss": 0.5327,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4899694919586182,
      "rewards/margins": 0.3111625611782074,
      "rewards/rejected": -1.8011322021484375,
      "sft_loss": 1.5551286935806274,
      "step": 1745
    },
    {
      "epoch": 0.9366114734905503,
      "grad_norm": 11.81033325638871,
      "learning_rate": 8.689979496279746e-07,
      "logits/chosen": -0.05830759555101395,
      "logits/rejected": 0.0124333705753088,
      "logps/chosen": -1.5286645889282227,
      "logps/rejected": -1.8843910694122314,
      "loss": 0.5324,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5286645889282227,
      "rewards/margins": 0.3557264506816864,
      "rewards/rejected": -1.8843910694122314,
      "sft_loss": 1.543145775794983,
      "step": 1750
    },
    {
      "epoch": 0.9392875062719518,
      "grad_norm": 5.7829093648908545,
      "learning_rate": 8.679452179074811e-07,
      "logits/chosen": -0.05018848180770874,
      "logits/rejected": 0.06818221509456635,
      "logps/chosen": -1.4554150104522705,
      "logps/rejected": -1.7889883518218994,
      "loss": 0.5154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4554150104522705,
      "rewards/margins": 0.33357328176498413,
      "rewards/rejected": -1.7889883518218994,
      "sft_loss": 1.4961915016174316,
      "step": 1755
    },
    {
      "epoch": 0.9419635390533534,
      "grad_norm": 7.000753850441391,
      "learning_rate": 8.668889163808698e-07,
      "logits/chosen": -0.031203657388687134,
      "logits/rejected": 0.10760529339313507,
      "logps/chosen": -1.415029764175415,
      "logps/rejected": -1.769468903541565,
      "loss": 0.5063,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.415029764175415,
      "rewards/margins": 0.3544391989707947,
      "rewards/rejected": -1.769468903541565,
      "sft_loss": 1.5223928689956665,
      "step": 1760
    },
    {
      "epoch": 0.944639571834755,
      "grad_norm": 6.464241093793835,
      "learning_rate": 8.658290552963827e-07,
      "logits/chosen": 0.033028002828359604,
      "logits/rejected": 0.058578938245773315,
      "logps/chosen": -1.4606726169586182,
      "logps/rejected": -1.8109041452407837,
      "loss": 0.5231,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4606726169586182,
      "rewards/margins": 0.35023149847984314,
      "rewards/rejected": -1.8109041452407837,
      "sft_loss": 1.487366795539856,
      "step": 1765
    },
    {
      "epoch": 0.9473156046161565,
      "grad_norm": 5.275683302776248,
      "learning_rate": 8.647656449367966e-07,
      "logits/chosen": 0.02882230281829834,
      "logits/rejected": 0.19418470561504364,
      "logps/chosen": -1.491390585899353,
      "logps/rejected": -1.716875433921814,
      "loss": 0.5343,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.491390585899353,
      "rewards/margins": 0.22548499703407288,
      "rewards/rejected": -1.716875433921814,
      "sft_loss": 1.5724704265594482,
      "step": 1770
    },
    {
      "epoch": 0.9499916373975581,
      "grad_norm": 5.048787352166431,
      "learning_rate": 8.636986956193235e-07,
      "logits/chosen": -0.05322617292404175,
      "logits/rejected": 0.053495001047849655,
      "logps/chosen": -1.4078679084777832,
      "logps/rejected": -1.7306697368621826,
      "loss": 0.5073,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4078679084777832,
      "rewards/margins": 0.32280176877975464,
      "rewards/rejected": -1.7306697368621826,
      "sft_loss": 1.4821839332580566,
      "step": 1775
    },
    {
      "epoch": 0.9526676701789597,
      "grad_norm": 6.082908963768483,
      "learning_rate": 8.626282176955104e-07,
      "logits/chosen": -0.05241629481315613,
      "logits/rejected": 0.08349411189556122,
      "logps/chosen": -1.3877642154693604,
      "logps/rejected": -1.7418733835220337,
      "loss": 0.4993,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.3877642154693604,
      "rewards/margins": 0.35410913825035095,
      "rewards/rejected": -1.7418733835220337,
      "sft_loss": 1.3849250078201294,
      "step": 1780
    },
    {
      "epoch": 0.9553437029603613,
      "grad_norm": 5.8152861408899605,
      "learning_rate": 8.615542215511389e-07,
      "logits/chosen": 0.07076708227396011,
      "logits/rejected": 0.1529882252216339,
      "logps/chosen": -1.3420288562774658,
      "logps/rejected": -1.5302860736846924,
      "loss": 0.5184,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.3420288562774658,
      "rewards/margins": 0.1882571130990982,
      "rewards/rejected": -1.5302860736846924,
      "sft_loss": 1.3648477792739868,
      "step": 1785
    },
    {
      "epoch": 0.9580197357417628,
      "grad_norm": 5.7034590119810185,
      "learning_rate": 8.604767176061241e-07,
      "logits/chosen": 0.06448648869991302,
      "logits/rejected": 0.12127365171909332,
      "logps/chosen": -1.4409104585647583,
      "logps/rejected": -1.6786985397338867,
      "loss": 0.5331,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4409104585647583,
      "rewards/margins": 0.2377881556749344,
      "rewards/rejected": -1.6786985397338867,
      "sft_loss": 1.49290931224823,
      "step": 1790
    },
    {
      "epoch": 0.9606957685231644,
      "grad_norm": 4.6440102903748,
      "learning_rate": 8.593957163144141e-07,
      "logits/chosen": -0.07223442196846008,
      "logits/rejected": 0.08344900608062744,
      "logps/chosen": -1.3767060041427612,
      "logps/rejected": -1.7289140224456787,
      "loss": 0.4995,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3767060041427612,
      "rewards/margins": 0.35220807790756226,
      "rewards/rejected": -1.7289140224456787,
      "sft_loss": 1.452522873878479,
      "step": 1795
    },
    {
      "epoch": 0.963371801304566,
      "grad_norm": 4.507598778221961,
      "learning_rate": 8.58311228163888e-07,
      "logits/chosen": -0.004403007682412863,
      "logits/rejected": 0.08500498533248901,
      "logps/chosen": -1.4097429513931274,
      "logps/rejected": -1.6911542415618896,
      "loss": 0.5172,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4097429513931274,
      "rewards/margins": 0.28141140937805176,
      "rewards/rejected": -1.6911542415618896,
      "sft_loss": 1.5189019441604614,
      "step": 1800
    },
    {
      "epoch": 0.9660478340859675,
      "grad_norm": 5.460169181811743,
      "learning_rate": 8.57223263676255e-07,
      "logits/chosen": -0.13715745508670807,
      "logits/rejected": 0.010128025896847248,
      "logps/chosen": -1.3865242004394531,
      "logps/rejected": -1.8770020008087158,
      "loss": 0.4638,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3865242004394531,
      "rewards/margins": 0.4904778003692627,
      "rewards/rejected": -1.8770020008087158,
      "sft_loss": 1.4646705389022827,
      "step": 1805
    },
    {
      "epoch": 0.9687238668673691,
      "grad_norm": 5.003102751583516,
      "learning_rate": 8.561318334069511e-07,
      "logits/chosen": 0.021493779495358467,
      "logits/rejected": 0.18332651257514954,
      "logps/chosen": -1.4104686975479126,
      "logps/rejected": -1.7444658279418945,
      "loss": 0.5138,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4104686975479126,
      "rewards/margins": 0.3339969515800476,
      "rewards/rejected": -1.7444658279418945,
      "sft_loss": 1.4638065099716187,
      "step": 1810
    },
    {
      "epoch": 0.9713998996487707,
      "grad_norm": 5.316062872583758,
      "learning_rate": 8.550369479450375e-07,
      "logits/chosen": -0.033848315477371216,
      "logits/rejected": 0.12872003018856049,
      "logps/chosen": -1.456559419631958,
      "logps/rejected": -1.7650468349456787,
      "loss": 0.5169,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.456559419631958,
      "rewards/margins": 0.3084874749183655,
      "rewards/rejected": -1.7650468349456787,
      "sft_loss": 1.5225870609283447,
      "step": 1815
    },
    {
      "epoch": 0.9740759324301723,
      "grad_norm": 6.469002214831444,
      "learning_rate": 8.539386179130977e-07,
      "logits/chosen": -0.008161703124642372,
      "logits/rejected": 0.07683838903903961,
      "logps/chosen": -1.4914897680282593,
      "logps/rejected": -1.796868085861206,
      "loss": 0.5163,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4914897680282593,
      "rewards/margins": 0.3053787052631378,
      "rewards/rejected": -1.796868085861206,
      "sft_loss": 1.4638153314590454,
      "step": 1820
    },
    {
      "epoch": 0.9767519652115738,
      "grad_norm": 6.337595333137927,
      "learning_rate": 8.528368539671347e-07,
      "logits/chosen": -0.08625678718090057,
      "logits/rejected": 0.08819940686225891,
      "logps/chosen": -1.4118739366531372,
      "logps/rejected": -1.8817695379257202,
      "loss": 0.5078,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4118739366531372,
      "rewards/margins": 0.46989554166793823,
      "rewards/rejected": -1.8817695379257202,
      "sft_loss": 1.483617901802063,
      "step": 1825
    },
    {
      "epoch": 0.9794279979929754,
      "grad_norm": 4.306246185410908,
      "learning_rate": 8.51731666796467e-07,
      "logits/chosen": 0.09946813434362411,
      "logits/rejected": 0.1469993144273758,
      "logps/chosen": -1.5145267248153687,
      "logps/rejected": -1.7509231567382812,
      "loss": 0.5364,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.5145267248153687,
      "rewards/margins": 0.2363964021205902,
      "rewards/rejected": -1.7509231567382812,
      "sft_loss": 1.5372315645217896,
      "step": 1830
    },
    {
      "epoch": 0.982104030774377,
      "grad_norm": 6.10389102120731,
      "learning_rate": 8.506230671236254e-07,
      "logits/chosen": -0.002360707614570856,
      "logits/rejected": 0.07914100587368011,
      "logps/chosen": -1.4556949138641357,
      "logps/rejected": -1.6280122995376587,
      "loss": 0.5561,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4556949138641357,
      "rewards/margins": 0.17231719195842743,
      "rewards/rejected": -1.6280122995376587,
      "sft_loss": 1.5284003019332886,
      "step": 1835
    },
    {
      "epoch": 0.9847800635557785,
      "grad_norm": 6.074650721812385,
      "learning_rate": 8.495110657042488e-07,
      "logits/chosen": 0.06208980083465576,
      "logits/rejected": 0.19643422961235046,
      "logps/chosen": -1.467151403427124,
      "logps/rejected": -1.8871538639068604,
      "loss": 0.4898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.467151403427124,
      "rewards/margins": 0.4200025200843811,
      "rewards/rejected": -1.8871538639068604,
      "sft_loss": 1.5178897380828857,
      "step": 1840
    },
    {
      "epoch": 0.9874560963371801,
      "grad_norm": 7.47352279179211,
      "learning_rate": 8.483956733269799e-07,
      "logits/chosen": -0.009203584864735603,
      "logits/rejected": 0.10609463602304459,
      "logps/chosen": -1.5118629932403564,
      "logps/rejected": -1.8410472869873047,
      "loss": 0.5666,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.5118629932403564,
      "rewards/margins": 0.3291842043399811,
      "rewards/rejected": -1.8410472869873047,
      "sft_loss": 1.5683891773223877,
      "step": 1845
    },
    {
      "epoch": 0.9901321291185817,
      "grad_norm": 12.76772038228278,
      "learning_rate": 8.472769008133602e-07,
      "logits/chosen": -0.1566140055656433,
      "logits/rejected": -0.010694952681660652,
      "logps/chosen": -1.558301568031311,
      "logps/rejected": -1.7742935419082642,
      "loss": 0.5796,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.558301568031311,
      "rewards/margins": 0.2159920185804367,
      "rewards/rejected": -1.7742935419082642,
      "sft_loss": 1.5489089488983154,
      "step": 1850
    },
    {
      "epoch": 0.9928081618999832,
      "grad_norm": 10.974762656119584,
      "learning_rate": 8.461547590177259e-07,
      "logits/chosen": -0.05199091508984566,
      "logits/rejected": 0.08179700374603271,
      "logps/chosen": -1.4593642950057983,
      "logps/rejected": -1.8170993328094482,
      "loss": 0.5136,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4593642950057983,
      "rewards/margins": 0.3577350974082947,
      "rewards/rejected": -1.8170993328094482,
      "sft_loss": 1.519140362739563,
      "step": 1855
    },
    {
      "epoch": 0.9954841946813848,
      "grad_norm": 8.61140192519221,
      "learning_rate": 8.450292588271014e-07,
      "logits/chosen": -0.03101665899157524,
      "logits/rejected": 0.06896068155765533,
      "logps/chosen": -1.592306137084961,
      "logps/rejected": -1.8628515005111694,
      "loss": 0.5535,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.592306137084961,
      "rewards/margins": 0.27054545283317566,
      "rewards/rejected": -1.8628515005111694,
      "sft_loss": 1.5682618618011475,
      "step": 1860
    },
    {
      "epoch": 0.9981602274627864,
      "grad_norm": 5.49984973042548,
      "learning_rate": 8.439004111610945e-07,
      "logits/chosen": -0.06608717143535614,
      "logits/rejected": 0.02036227658390999,
      "logps/chosen": -1.3796789646148682,
      "logps/rejected": -1.7269248962402344,
      "loss": 0.5016,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3796789646148682,
      "rewards/margins": 0.3472460210323334,
      "rewards/rejected": -1.7269248962402344,
      "sft_loss": 1.4405910968780518,
      "step": 1865
    },
    {
      "epoch": 1.000836260244188,
      "grad_norm": 8.509372779398811,
      "learning_rate": 8.427682269717901e-07,
      "logits/chosen": -0.0892179012298584,
      "logits/rejected": 0.0769272893667221,
      "logps/chosen": -1.5132403373718262,
      "logps/rejected": -1.8297183513641357,
      "loss": 0.5254,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.5132403373718262,
      "rewards/margins": 0.31647807359695435,
      "rewards/rejected": -1.8297183513641357,
      "sft_loss": 1.5695701837539673,
      "step": 1870
    },
    {
      "epoch": 1.0035122930255895,
      "grad_norm": 10.940379009851805,
      "learning_rate": 8.416327172436446e-07,
      "logits/chosen": -0.13610494136810303,
      "logits/rejected": 0.0053990124724805355,
      "logps/chosen": -1.499241590499878,
      "logps/rejected": -1.732712984085083,
      "loss": 0.5484,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.499241590499878,
      "rewards/margins": 0.23347143828868866,
      "rewards/rejected": -1.732712984085083,
      "sft_loss": 1.4607391357421875,
      "step": 1875
    },
    {
      "epoch": 1.0061883258069912,
      "grad_norm": 8.276979456989288,
      "learning_rate": 8.404938929933778e-07,
      "logits/chosen": -0.018297823145985603,
      "logits/rejected": 0.16213171184062958,
      "logps/chosen": -1.4397025108337402,
      "logps/rejected": -1.926867127418518,
      "loss": 0.487,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4397025108337402,
      "rewards/margins": 0.4871644973754883,
      "rewards/rejected": -1.926867127418518,
      "sft_loss": 1.5095751285552979,
      "step": 1880
    },
    {
      "epoch": 1.0088643585883927,
      "grad_norm": 7.535703865238154,
      "learning_rate": 8.39351765269868e-07,
      "logits/chosen": -0.07311379909515381,
      "logits/rejected": 0.011505231261253357,
      "logps/chosen": -1.3841418027877808,
      "logps/rejected": -1.7198749780654907,
      "loss": 0.5189,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3841418027877808,
      "rewards/margins": 0.33573317527770996,
      "rewards/rejected": -1.7198749780654907,
      "sft_loss": 1.3955576419830322,
      "step": 1885
    },
    {
      "epoch": 1.0115403913697942,
      "grad_norm": 6.248961178342216,
      "learning_rate": 8.382063451540431e-07,
      "logits/chosen": -0.07998314499855042,
      "logits/rejected": 0.127044677734375,
      "logps/chosen": -1.4475855827331543,
      "logps/rejected": -1.784197211265564,
      "loss": 0.4888,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4475855827331543,
      "rewards/margins": 0.33661171793937683,
      "rewards/rejected": -1.784197211265564,
      "sft_loss": 1.5629757642745972,
      "step": 1890
    },
    {
      "epoch": 1.014216424151196,
      "grad_norm": 5.787207779107956,
      "learning_rate": 8.370576437587742e-07,
      "logits/chosen": -0.042304106056690216,
      "logits/rejected": 0.019032040610909462,
      "logps/chosen": -1.4489058256149292,
      "logps/rejected": -1.7134273052215576,
      "loss": 0.5019,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4489058256149292,
      "rewards/margins": 0.2645213305950165,
      "rewards/rejected": -1.7134273052215576,
      "sft_loss": 1.475355625152588,
      "step": 1895
    },
    {
      "epoch": 1.0168924569325974,
      "grad_norm": 5.317094749287823,
      "learning_rate": 8.359056722287674e-07,
      "logits/chosen": -0.1466391235589981,
      "logits/rejected": 0.12817367911338806,
      "logps/chosen": -1.4559046030044556,
      "logps/rejected": -1.783931016921997,
      "loss": 0.492,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4559046030044556,
      "rewards/margins": 0.32802626490592957,
      "rewards/rejected": -1.783931016921997,
      "sft_loss": 1.5369385480880737,
      "step": 1900
    },
    {
      "epoch": 1.019568489713999,
      "grad_norm": 3.8154740406992573,
      "learning_rate": 8.347504417404553e-07,
      "logits/chosen": -0.07074271142482758,
      "logits/rejected": 0.09242469817399979,
      "logps/chosen": -1.4736456871032715,
      "logps/rejected": -1.7548481225967407,
      "loss": 0.5259,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4736456871032715,
      "rewards/margins": 0.28120237588882446,
      "rewards/rejected": -1.7548481225967407,
      "sft_loss": 1.4908063411712646,
      "step": 1905
    },
    {
      "epoch": 1.0222445224954007,
      "grad_norm": 5.561914424996803,
      "learning_rate": 8.335919635018893e-07,
      "logits/chosen": -0.15970030426979065,
      "logits/rejected": -0.026617711409926414,
      "logps/chosen": -1.450312852859497,
      "logps/rejected": -1.7166593074798584,
      "loss": 0.5145,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.450312852859497,
      "rewards/margins": 0.26634639501571655,
      "rewards/rejected": -1.7166593074798584,
      "sft_loss": 1.5004311800003052,
      "step": 1910
    },
    {
      "epoch": 1.0249205552768021,
      "grad_norm": 4.962043417804353,
      "learning_rate": 8.324302487526303e-07,
      "logits/chosen": -0.10115758329629898,
      "logits/rejected": -0.015957407653331757,
      "logps/chosen": -1.4378993511199951,
      "logps/rejected": -1.6727068424224854,
      "loss": 0.532,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4378993511199951,
      "rewards/margins": 0.23480752110481262,
      "rewards/rejected": -1.6727068424224854,
      "sft_loss": 1.531611442565918,
      "step": 1915
    },
    {
      "epoch": 1.0275965880582036,
      "grad_norm": 4.892120380898045,
      "learning_rate": 8.312653087636398e-07,
      "logits/chosen": -0.14030539989471436,
      "logits/rejected": -0.05974206328392029,
      "logps/chosen": -1.324191689491272,
      "logps/rejected": -1.6707344055175781,
      "loss": 0.4798,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.324191689491272,
      "rewards/margins": 0.346542626619339,
      "rewards/rejected": -1.6707344055175781,
      "sft_loss": 1.3921024799346924,
      "step": 1920
    },
    {
      "epoch": 1.0302726208396054,
      "grad_norm": 7.148324764965389,
      "learning_rate": 8.300971548371711e-07,
      "logits/chosen": -0.2657248377799988,
      "logits/rejected": -0.05455995723605156,
      "logps/chosen": -1.4857937097549438,
      "logps/rejected": -1.766405701637268,
      "loss": 0.5245,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4857937097549438,
      "rewards/margins": 0.280612051486969,
      "rewards/rejected": -1.766405701637268,
      "sft_loss": 1.5520381927490234,
      "step": 1925
    },
    {
      "epoch": 1.0329486536210069,
      "grad_norm": 7.551954007824476,
      "learning_rate": 8.289257983066582e-07,
      "logits/chosen": -0.13647429645061493,
      "logits/rejected": 0.00663360208272934,
      "logps/chosen": -1.36500084400177,
      "logps/rejected": -1.7560393810272217,
      "loss": 0.4665,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.36500084400177,
      "rewards/margins": 0.391038715839386,
      "rewards/rejected": -1.7560393810272217,
      "sft_loss": 1.4452471733093262,
      "step": 1930
    },
    {
      "epoch": 1.0356246864024083,
      "grad_norm": 6.0992843581974405,
      "learning_rate": 8.277512505366077e-07,
      "logits/chosen": -0.19032074511051178,
      "logits/rejected": 0.009153047576546669,
      "logps/chosen": -1.4787334203720093,
      "logps/rejected": -1.8719135522842407,
      "loss": 0.4813,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4787334203720093,
      "rewards/margins": 0.39318010210990906,
      "rewards/rejected": -1.8719135522842407,
      "sft_loss": 1.5212489366531372,
      "step": 1935
    },
    {
      "epoch": 1.03830071918381,
      "grad_norm": 5.350166802718159,
      "learning_rate": 8.265735229224868e-07,
      "logits/chosen": -0.0966397374868393,
      "logits/rejected": 0.03161340951919556,
      "logps/chosen": -1.4548650979995728,
      "logps/rejected": -1.8944162130355835,
      "loss": 0.4747,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4548650979995728,
      "rewards/margins": 0.4395512640476227,
      "rewards/rejected": -1.8944162130355835,
      "sft_loss": 1.4918707609176636,
      "step": 1940
    },
    {
      "epoch": 1.0409767519652116,
      "grad_norm": 4.2595610579632615,
      "learning_rate": 8.253926268906144e-07,
      "logits/chosen": -0.20996013283729553,
      "logits/rejected": -0.03253777697682381,
      "logps/chosen": -1.4889169931411743,
      "logps/rejected": -2.000814437866211,
      "loss": 0.4763,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4889169931411743,
      "rewards/margins": 0.5118975639343262,
      "rewards/rejected": -2.000814437866211,
      "sft_loss": 1.5587809085845947,
      "step": 1945
    },
    {
      "epoch": 1.043652784746613,
      "grad_norm": 6.367327140932028,
      "learning_rate": 8.242085738980487e-07,
      "logits/chosen": -0.12788192927837372,
      "logits/rejected": 0.09394869208335876,
      "logps/chosen": -1.5442017316818237,
      "logps/rejected": -1.9254404306411743,
      "loss": 0.5055,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5442017316818237,
      "rewards/margins": 0.38123857975006104,
      "rewards/rejected": -1.9254404306411743,
      "sft_loss": 1.6286182403564453,
      "step": 1950
    },
    {
      "epoch": 1.0463288175280148,
      "grad_norm": 4.895652101621527,
      "learning_rate": 8.230213754324772e-07,
      "logits/chosen": -0.11432129144668579,
      "logits/rejected": -0.04440194368362427,
      "logps/chosen": -1.4022363424301147,
      "logps/rejected": -1.7887109518051147,
      "loss": 0.4738,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4022363424301147,
      "rewards/margins": 0.3864745497703552,
      "rewards/rejected": -1.7887109518051147,
      "sft_loss": 1.4586220979690552,
      "step": 1955
    },
    {
      "epoch": 1.0490048503094163,
      "grad_norm": 5.74006277179219,
      "learning_rate": 8.218310430121045e-07,
      "logits/chosen": -0.18158350884914398,
      "logits/rejected": -0.14121797680854797,
      "logps/chosen": -1.4819562435150146,
      "logps/rejected": -1.8232675790786743,
      "loss": 0.5088,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4819562435150146,
      "rewards/margins": 0.3413112759590149,
      "rewards/rejected": -1.8232675790786743,
      "sft_loss": 1.5612026453018188,
      "step": 1960
    },
    {
      "epoch": 1.051680883090818,
      "grad_norm": 5.81208081953113,
      "learning_rate": 8.20637588185541e-07,
      "logits/chosen": -0.0955495834350586,
      "logits/rejected": -0.02487076446413994,
      "logps/chosen": -1.397090196609497,
      "logps/rejected": -1.9855940341949463,
      "loss": 0.4492,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.397090196609497,
      "rewards/margins": 0.588503897190094,
      "rewards/rejected": -1.9855940341949463,
      "sft_loss": 1.4849728345870972,
      "step": 1965
    },
    {
      "epoch": 1.0543569158722195,
      "grad_norm": 4.5388641581334355,
      "learning_rate": 8.194410225316906e-07,
      "logits/chosen": -0.13712339103221893,
      "logits/rejected": 0.014775209128856659,
      "logps/chosen": -1.4033243656158447,
      "logps/rejected": -1.7691576480865479,
      "loss": 0.4902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4033243656158447,
      "rewards/margins": 0.36583322286605835,
      "rewards/rejected": -1.7691576480865479,
      "sft_loss": 1.4541233777999878,
      "step": 1970
    },
    {
      "epoch": 1.057032948653621,
      "grad_norm": 6.767756108003455,
      "learning_rate": 8.182413576596385e-07,
      "logits/chosen": -0.05116777494549751,
      "logits/rejected": 0.04594787210226059,
      "logps/chosen": -1.345689296722412,
      "logps/rejected": -1.7165015935897827,
      "loss": 0.4719,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.345689296722412,
      "rewards/margins": 0.37081241607666016,
      "rewards/rejected": -1.7165015935897827,
      "sft_loss": 1.4144160747528076,
      "step": 1975
    },
    {
      "epoch": 1.0597089814350227,
      "grad_norm": 5.729373646531792,
      "learning_rate": 8.170386052085389e-07,
      "logits/chosen": -0.02618979476392269,
      "logits/rejected": 0.10099951922893524,
      "logps/chosen": -1.4654579162597656,
      "logps/rejected": -1.8436918258666992,
      "loss": 0.5143,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.4654579162597656,
      "rewards/margins": 0.37823402881622314,
      "rewards/rejected": -1.8436918258666992,
      "sft_loss": 1.4997092485427856,
      "step": 1980
    },
    {
      "epoch": 1.0623850142164242,
      "grad_norm": 6.710935394624452,
      "learning_rate": 8.158327768475008e-07,
      "logits/chosen": -0.09092311561107635,
      "logits/rejected": 0.07339204847812653,
      "logps/chosen": -1.4836839437484741,
      "logps/rejected": -1.7372486591339111,
      "loss": 0.5376,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4836839437484741,
      "rewards/margins": 0.2535645067691803,
      "rewards/rejected": -1.7372486591339111,
      "sft_loss": 1.4888941049575806,
      "step": 1985
    },
    {
      "epoch": 1.0650610469978257,
      "grad_norm": 7.69503470863718,
      "learning_rate": 8.146238842754767e-07,
      "logits/chosen": -0.12186932563781738,
      "logits/rejected": -0.016661647707223892,
      "logps/chosen": -1.4644496440887451,
      "logps/rejected": -1.7583051919937134,
      "loss": 0.5217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4644496440887451,
      "rewards/margins": 0.29385560750961304,
      "rewards/rejected": -1.7583051919937134,
      "sft_loss": 1.4940369129180908,
      "step": 1990
    },
    {
      "epoch": 1.0677370797792274,
      "grad_norm": 4.511316509520965,
      "learning_rate": 8.134119392211476e-07,
      "logits/chosen": -0.00946495495736599,
      "logits/rejected": 0.15335066616535187,
      "logps/chosen": -1.3617419004440308,
      "logps/rejected": -1.8576990365982056,
      "loss": 0.4515,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3617419004440308,
      "rewards/margins": 0.4959571957588196,
      "rewards/rejected": -1.8576990365982056,
      "sft_loss": 1.4200398921966553,
      "step": 1995
    },
    {
      "epoch": 1.0704131125606289,
      "grad_norm": 8.821228426339408,
      "learning_rate": 8.121969534428094e-07,
      "logits/chosen": -0.1066073551774025,
      "logits/rejected": 0.062150947749614716,
      "logps/chosen": -1.5116338729858398,
      "logps/rejected": -1.8206783533096313,
      "loss": 0.5743,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.5116338729858398,
      "rewards/margins": 0.30904462933540344,
      "rewards/rejected": -1.8206783533096313,
      "sft_loss": 1.5308339595794678,
      "step": 2000
    },
    {
      "epoch": 1.0704131125606289,
      "eval_logits/chosen": 0.21312709152698517,
      "eval_logits/rejected": 0.3106827437877655,
      "eval_logps/chosen": -1.433026909828186,
      "eval_logps/rejected": -1.807220697402954,
      "eval_loss": 0.5046906471252441,
      "eval_rewards/accuracies": 0.6305637955665588,
      "eval_rewards/chosen": -1.433026909828186,
      "eval_rewards/margins": 0.3741937279701233,
      "eval_rewards/rejected": -1.807220697402954,
      "eval_runtime": 43.4327,
      "eval_samples_per_second": 30.967,
      "eval_sft_loss": 1.488387107849121,
      "eval_steps_per_second": 7.759,
      "step": 2000
    },
    {
      "epoch": 1.0730891453420304,
      "grad_norm": 6.087055779041788,
      "learning_rate": 8.109789387282599e-07,
      "logits/chosen": -0.06976554542779922,
      "logits/rejected": 0.0286190714687109,
      "logps/chosen": -1.464482069015503,
      "logps/rejected": -1.7185428142547607,
      "loss": 0.55,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.464482069015503,
      "rewards/margins": 0.25406089425086975,
      "rewards/rejected": -1.7185428142547607,
      "sft_loss": 1.5256202220916748,
      "step": 2005
    },
    {
      "epoch": 1.075765178123432,
      "grad_norm": 6.057486499113076,
      "learning_rate": 8.097579068946827e-07,
      "logits/chosen": -0.018949469551444054,
      "logits/rejected": 0.09878715872764587,
      "logps/chosen": -1.3560127019882202,
      "logps/rejected": -1.6560802459716797,
      "loss": 0.493,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.3560127019882202,
      "rewards/margins": 0.3000675141811371,
      "rewards/rejected": -1.6560802459716797,
      "sft_loss": 1.4456104040145874,
      "step": 2010
    },
    {
      "epoch": 1.0784412109048336,
      "grad_norm": 7.26309234115051,
      "learning_rate": 8.085338697885344e-07,
      "logits/chosen": -0.08260910212993622,
      "logits/rejected": 0.07246457785367966,
      "logps/chosen": -1.3358352184295654,
      "logps/rejected": -1.6958932876586914,
      "loss": 0.4799,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.3358352184295654,
      "rewards/margins": 0.3600581884384155,
      "rewards/rejected": -1.6958932876586914,
      "sft_loss": 1.35738205909729,
      "step": 2015
    },
    {
      "epoch": 1.081117243686235,
      "grad_norm": 5.049206973817521,
      "learning_rate": 8.073068392854282e-07,
      "logits/chosen": -0.15729498863220215,
      "logits/rejected": 0.04396899417042732,
      "logps/chosen": -1.4487788677215576,
      "logps/rejected": -1.8310155868530273,
      "loss": 0.4963,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4487788677215576,
      "rewards/margins": 0.38223665952682495,
      "rewards/rejected": -1.8310155868530273,
      "sft_loss": 1.454693078994751,
      "step": 2020
    },
    {
      "epoch": 1.0837932764676368,
      "grad_norm": 5.494973211998128,
      "learning_rate": 8.060768272900193e-07,
      "logits/chosen": -0.03109714947640896,
      "logits/rejected": 0.11418769508600235,
      "logps/chosen": -1.4256064891815186,
      "logps/rejected": -1.8667850494384766,
      "loss": 0.4885,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4256064891815186,
      "rewards/margins": 0.4411783218383789,
      "rewards/rejected": -1.8667850494384766,
      "sft_loss": 1.5188822746276855,
      "step": 2025
    },
    {
      "epoch": 1.0864693092490383,
      "grad_norm": 4.978243268370478,
      "learning_rate": 8.0484384573589e-07,
      "logits/chosen": -0.12496711313724518,
      "logits/rejected": -0.09452895820140839,
      "logps/chosen": -1.3800010681152344,
      "logps/rejected": -1.6710617542266846,
      "loss": 0.5111,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.3800010681152344,
      "rewards/margins": 0.29106056690216064,
      "rewards/rejected": -1.6710617542266846,
      "sft_loss": 1.462308645248413,
      "step": 2030
    },
    {
      "epoch": 1.0891453420304398,
      "grad_norm": 7.69197040790886,
      "learning_rate": 8.03607906585432e-07,
      "logits/chosen": -0.1336095780134201,
      "logits/rejected": 0.051111944019794464,
      "logps/chosen": -1.4296190738677979,
      "logps/rejected": -1.8351964950561523,
      "loss": 0.505,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4296190738677979,
      "rewards/margins": 0.4055773615837097,
      "rewards/rejected": -1.8351964950561523,
      "sft_loss": 1.469069242477417,
      "step": 2035
    },
    {
      "epoch": 1.0918213748118415,
      "grad_norm": 33.53833279342883,
      "learning_rate": 8.023690218297329e-07,
      "logits/chosen": -0.21496620774269104,
      "logits/rejected": -0.13631033897399902,
      "logps/chosen": -1.4748858213424683,
      "logps/rejected": -1.791357398033142,
      "loss": 0.5437,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4748858213424683,
      "rewards/margins": 0.3164716958999634,
      "rewards/rejected": -1.791357398033142,
      "sft_loss": 1.5261235237121582,
      "step": 2040
    },
    {
      "epoch": 1.094497407593243,
      "grad_norm": 7.422760096027858,
      "learning_rate": 8.01127203488458e-07,
      "logits/chosen": -0.07252940535545349,
      "logits/rejected": -0.029904400929808617,
      "logps/chosen": -1.4626128673553467,
      "logps/rejected": -1.828600525856018,
      "loss": 0.4959,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4626128673553467,
      "rewards/margins": 0.3659875988960266,
      "rewards/rejected": -1.828600525856018,
      "sft_loss": 1.4483476877212524,
      "step": 2045
    },
    {
      "epoch": 1.0971734403746445,
      "grad_norm": 6.7260498381113125,
      "learning_rate": 7.998824636097339e-07,
      "logits/chosen": -0.19608174264431,
      "logits/rejected": -0.0628303661942482,
      "logps/chosen": -1.4419586658477783,
      "logps/rejected": -1.7460864782333374,
      "loss": 0.5296,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4419586658477783,
      "rewards/margins": 0.3041277527809143,
      "rewards/rejected": -1.7460864782333374,
      "sft_loss": 1.54558527469635,
      "step": 2050
    },
    {
      "epoch": 1.0998494731560462,
      "grad_norm": 5.623724890193116,
      "learning_rate": 7.986348142700328e-07,
      "logits/chosen": -0.126641184091568,
      "logits/rejected": 0.009659910574555397,
      "logps/chosen": -1.4176418781280518,
      "logps/rejected": -1.8081547021865845,
      "loss": 0.4977,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4176418781280518,
      "rewards/margins": 0.39051297307014465,
      "rewards/rejected": -1.8081547021865845,
      "sft_loss": 1.5018881559371948,
      "step": 2055
    },
    {
      "epoch": 1.1025255059374477,
      "grad_norm": 5.426254467551804,
      "learning_rate": 7.973842675740539e-07,
      "logits/chosen": -0.08474062383174896,
      "logits/rejected": -0.029642928391695023,
      "logps/chosen": -1.4769197702407837,
      "logps/rejected": -1.9251235723495483,
      "loss": 0.4868,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4769197702407837,
      "rewards/margins": 0.4482037127017975,
      "rewards/rejected": -1.9251235723495483,
      "sft_loss": 1.5756006240844727,
      "step": 2060
    },
    {
      "epoch": 1.1052015387188494,
      "grad_norm": 5.875803853615592,
      "learning_rate": 7.961308356546066e-07,
      "logits/chosen": -0.11285390704870224,
      "logits/rejected": 0.02270979806780815,
      "logps/chosen": -1.445842981338501,
      "logps/rejected": -1.824823021888733,
      "loss": 0.5066,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.445842981338501,
      "rewards/margins": 0.37897998094558716,
      "rewards/rejected": -1.824823021888733,
      "sft_loss": 1.4722983837127686,
      "step": 2065
    },
    {
      "epoch": 1.107877571500251,
      "grad_norm": 7.160997591730322,
      "learning_rate": 7.948745306724931e-07,
      "logits/chosen": -0.15063437819480896,
      "logits/rejected": -0.0021080286242067814,
      "logps/chosen": -1.3538507223129272,
      "logps/rejected": -1.8220882415771484,
      "loss": 0.4536,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3538507223129272,
      "rewards/margins": 0.4682373106479645,
      "rewards/rejected": -1.8220882415771484,
      "sft_loss": 1.4053022861480713,
      "step": 2070
    },
    {
      "epoch": 1.1105536042816524,
      "grad_norm": 7.926408403181835,
      "learning_rate": 7.936153648163897e-07,
      "logits/chosen": -0.16424348950386047,
      "logits/rejected": -0.05737115070223808,
      "logps/chosen": -1.4753576517105103,
      "logps/rejected": -1.9165945053100586,
      "loss": 0.4953,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4753576517105103,
      "rewards/margins": 0.44123712182044983,
      "rewards/rejected": -1.9165945053100586,
      "sft_loss": 1.5894849300384521,
      "step": 2075
    },
    {
      "epoch": 1.1132296370630541,
      "grad_norm": 5.00403006548638,
      "learning_rate": 7.92353350302729e-07,
      "logits/chosen": -0.23114259541034698,
      "logits/rejected": -0.07610608637332916,
      "logps/chosen": -1.3857789039611816,
      "logps/rejected": -1.8070179224014282,
      "loss": 0.4581,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3857789039611816,
      "rewards/margins": 0.4212391972541809,
      "rewards/rejected": -1.8070179224014282,
      "sft_loss": 1.4718667268753052,
      "step": 2080
    },
    {
      "epoch": 1.1159056698444556,
      "grad_norm": 9.039232195890655,
      "learning_rate": 7.910884993755816e-07,
      "logits/chosen": -0.18809795379638672,
      "logits/rejected": -0.07240627706050873,
      "logps/chosen": -1.4653651714324951,
      "logps/rejected": -1.8892971277236938,
      "loss": 0.4885,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4653651714324951,
      "rewards/margins": 0.4239319860935211,
      "rewards/rejected": -1.8892971277236938,
      "sft_loss": 1.5039739608764648,
      "step": 2085
    },
    {
      "epoch": 1.118581702625857,
      "grad_norm": 6.0073439848626835,
      "learning_rate": 7.898208243065367e-07,
      "logits/chosen": -0.2471090853214264,
      "logits/rejected": -0.23510317504405975,
      "logps/chosen": -1.439173698425293,
      "logps/rejected": -1.6933097839355469,
      "loss": 0.5324,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.439173698425293,
      "rewards/margins": 0.2541360557079315,
      "rewards/rejected": -1.6933097839355469,
      "sft_loss": 1.6191877126693726,
      "step": 2090
    },
    {
      "epoch": 1.1212577354072588,
      "grad_norm": 6.183660125167922,
      "learning_rate": 7.88550337394583e-07,
      "logits/chosen": -0.1707983911037445,
      "logits/rejected": -0.020055923610925674,
      "logps/chosen": -1.6324489116668701,
      "logps/rejected": -1.9510940313339233,
      "loss": 0.5457,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.6324489116668701,
      "rewards/margins": 0.31864503026008606,
      "rewards/rejected": -1.9510940313339233,
      "sft_loss": 1.6711390018463135,
      "step": 2095
    },
    {
      "epoch": 1.1239337681886603,
      "grad_norm": 7.045846864225031,
      "learning_rate": 7.872770509659905e-07,
      "logits/chosen": -0.08161846548318863,
      "logits/rejected": -0.026478594169020653,
      "logps/chosen": -1.6238008737564087,
      "logps/rejected": -1.9390065670013428,
      "loss": 0.5375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.6238008737564087,
      "rewards/margins": 0.31520575284957886,
      "rewards/rejected": -1.9390065670013428,
      "sft_loss": 1.637861967086792,
      "step": 2100
    },
    {
      "epoch": 1.1266098009700618,
      "grad_norm": 8.244391342508212,
      "learning_rate": 7.860009773741896e-07,
      "logits/chosen": -0.06960280239582062,
      "logits/rejected": 0.05846409872174263,
      "logps/chosen": -1.513601303100586,
      "logps/rejected": -1.9503761529922485,
      "loss": 0.4993,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.513601303100586,
      "rewards/margins": 0.4367748200893402,
      "rewards/rejected": -1.9503761529922485,
      "sft_loss": 1.5123342275619507,
      "step": 2105
    },
    {
      "epoch": 1.1292858337514635,
      "grad_norm": 9.673424557945967,
      "learning_rate": 7.84722128999652e-07,
      "logits/chosen": -0.18066242337226868,
      "logits/rejected": -0.017239436507225037,
      "logps/chosen": -1.4642972946166992,
      "logps/rejected": -2.0365207195281982,
      "loss": 0.49,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4642972946166992,
      "rewards/margins": 0.5722236633300781,
      "rewards/rejected": -2.0365207195281982,
      "sft_loss": 1.5431797504425049,
      "step": 2110
    },
    {
      "epoch": 1.131961866532865,
      "grad_norm": 5.776090992074557,
      "learning_rate": 7.834405182497699e-07,
      "logits/chosen": -0.023362573236227036,
      "logits/rejected": 0.043627046048641205,
      "logps/chosen": -1.438981294631958,
      "logps/rejected": -1.8212578296661377,
      "loss": 0.5045,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.438981294631958,
      "rewards/margins": 0.3822764754295349,
      "rewards/rejected": -1.8212578296661377,
      "sft_loss": 1.5206745862960815,
      "step": 2115
    },
    {
      "epoch": 1.1346378993142665,
      "grad_norm": 6.004133174670174,
      "learning_rate": 7.821561575587368e-07,
      "logits/chosen": -0.16063198447227478,
      "logits/rejected": -0.11137296259403229,
      "logps/chosen": -1.452800989151001,
      "logps/rejected": -1.7701566219329834,
      "loss": 0.5031,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.452800989151001,
      "rewards/margins": 0.3173556923866272,
      "rewards/rejected": -1.7701566219329834,
      "sft_loss": 1.5503771305084229,
      "step": 2120
    },
    {
      "epoch": 1.1373139320956682,
      "grad_norm": 5.022632822038148,
      "learning_rate": 7.808690593874254e-07,
      "logits/chosen": -0.12830975651741028,
      "logits/rejected": -0.06951282173395157,
      "logps/chosen": -1.4189093112945557,
      "logps/rejected": -1.8615138530731201,
      "loss": 0.4625,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4189093112945557,
      "rewards/margins": 0.4426047205924988,
      "rewards/rejected": -1.8615138530731201,
      "sft_loss": 1.4784224033355713,
      "step": 2125
    },
    {
      "epoch": 1.1399899648770697,
      "grad_norm": 8.589342131042367,
      "learning_rate": 7.79579236223268e-07,
      "logits/chosen": -0.09218287467956543,
      "logits/rejected": 0.16176007688045502,
      "logps/chosen": -1.4292819499969482,
      "logps/rejected": -1.8710426092147827,
      "loss": 0.4797,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4292819499969482,
      "rewards/margins": 0.44176048040390015,
      "rewards/rejected": -1.8710426092147827,
      "sft_loss": 1.517459750175476,
      "step": 2130
    },
    {
      "epoch": 1.1426659976584714,
      "grad_norm": 4.69038184236173,
      "learning_rate": 7.782867005801346e-07,
      "logits/chosen": -0.07629652321338654,
      "logits/rejected": 0.09444563090801239,
      "logps/chosen": -1.4645318984985352,
      "logps/rejected": -1.9413913488388062,
      "loss": 0.4812,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4645318984985352,
      "rewards/margins": 0.47685950994491577,
      "rewards/rejected": -1.9413913488388062,
      "sft_loss": 1.5025291442871094,
      "step": 2135
    },
    {
      "epoch": 1.145342030439873,
      "grad_norm": 9.283361039972426,
      "learning_rate": 7.769914649982117e-07,
      "logits/chosen": -0.10875393450260162,
      "logits/rejected": 0.03513630852103233,
      "logps/chosen": -1.4688737392425537,
      "logps/rejected": -1.8724868297576904,
      "loss": 0.5058,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4688737392425537,
      "rewards/margins": 0.40361303091049194,
      "rewards/rejected": -1.8724868297576904,
      "sft_loss": 1.504885196685791,
      "step": 2140
    },
    {
      "epoch": 1.1480180632212744,
      "grad_norm": 6.486958122860878,
      "learning_rate": 7.756935420438803e-07,
      "logits/chosen": -0.07364483177661896,
      "logits/rejected": 0.024317726492881775,
      "logps/chosen": -1.3688395023345947,
      "logps/rejected": -1.88224196434021,
      "loss": 0.457,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3688395023345947,
      "rewards/margins": 0.5134025812149048,
      "rewards/rejected": -1.88224196434021,
      "sft_loss": 1.4580392837524414,
      "step": 2145
    },
    {
      "epoch": 1.1506940960026761,
      "grad_norm": 5.235101078670038,
      "learning_rate": 7.743929443095951e-07,
      "logits/chosen": -0.14383387565612793,
      "logits/rejected": -0.08119402080774307,
      "logps/chosen": -1.5173956155776978,
      "logps/rejected": -1.9180179834365845,
      "loss": 0.4963,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5173956155776978,
      "rewards/margins": 0.40062230825424194,
      "rewards/rejected": -1.9180179834365845,
      "sft_loss": 1.5680986642837524,
      "step": 2150
    },
    {
      "epoch": 1.1533701287840776,
      "grad_norm": 6.894203466794916,
      "learning_rate": 7.730896844137609e-07,
      "logits/chosen": -0.06564654409885406,
      "logits/rejected": 0.009191582910716534,
      "logps/chosen": -1.5647809505462646,
      "logps/rejected": -1.9199464321136475,
      "loss": 0.5193,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5647809505462646,
      "rewards/margins": 0.35516566038131714,
      "rewards/rejected": -1.9199464321136475,
      "sft_loss": 1.6604130268096924,
      "step": 2155
    },
    {
      "epoch": 1.1560461615654791,
      "grad_norm": 7.756494004043538,
      "learning_rate": 7.717837750006106e-07,
      "logits/chosen": -0.15478457510471344,
      "logits/rejected": -0.04507964476943016,
      "logps/chosen": -1.4633822441101074,
      "logps/rejected": -1.9364042282104492,
      "loss": 0.4843,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4633822441101074,
      "rewards/margins": 0.47302207350730896,
      "rewards/rejected": -1.9364042282104492,
      "sft_loss": 1.5360000133514404,
      "step": 2160
    },
    {
      "epoch": 1.1587221943468808,
      "grad_norm": 5.616002529498294,
      "learning_rate": 7.704752287400832e-07,
      "logits/chosen": -0.1120419129729271,
      "logits/rejected": 0.08104712516069412,
      "logps/chosen": -1.493404507637024,
      "logps/rejected": -1.9760732650756836,
      "loss": 0.4831,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.493404507637024,
      "rewards/margins": 0.4826684594154358,
      "rewards/rejected": -1.9760732650756836,
      "sft_loss": 1.5499591827392578,
      "step": 2165
    },
    {
      "epoch": 1.1613982271282823,
      "grad_norm": 4.46886017298503,
      "learning_rate": 7.691640583277004e-07,
      "logits/chosen": -0.1037856787443161,
      "logits/rejected": 0.07713104784488678,
      "logps/chosen": -1.4457694292068481,
      "logps/rejected": -1.9311643838882446,
      "loss": 0.4887,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.4457694292068481,
      "rewards/margins": 0.4853949546813965,
      "rewards/rejected": -1.9311643838882446,
      "sft_loss": 1.5152101516723633,
      "step": 2170
    },
    {
      "epoch": 1.1640742599096838,
      "grad_norm": 4.794226804069966,
      "learning_rate": 7.678502764844433e-07,
      "logits/chosen": -0.12479491531848907,
      "logits/rejected": 0.06079835817217827,
      "logps/chosen": -1.502943754196167,
      "logps/rejected": -1.8639252185821533,
      "loss": 0.4995,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.502943754196167,
      "rewards/margins": 0.36098119616508484,
      "rewards/rejected": -1.8639252185821533,
      "sft_loss": 1.5932748317718506,
      "step": 2175
    },
    {
      "epoch": 1.1667502926910855,
      "grad_norm": 4.57957758743556,
      "learning_rate": 7.665338959566288e-07,
      "logits/chosen": -0.13211789727210999,
      "logits/rejected": -0.04276316240429878,
      "logps/chosen": -1.4503905773162842,
      "logps/rejected": -1.8833653926849365,
      "loss": 0.4839,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4503905773162842,
      "rewards/margins": 0.432974636554718,
      "rewards/rejected": -1.8833653926849365,
      "sft_loss": 1.5527359247207642,
      "step": 2180
    },
    {
      "epoch": 1.169426325472487,
      "grad_norm": 7.278664056579846,
      "learning_rate": 7.652149295157868e-07,
      "logits/chosen": -0.057872939854860306,
      "logits/rejected": 0.09013696759939194,
      "logps/chosen": -1.4882774353027344,
      "logps/rejected": -1.8309959173202515,
      "loss": 0.5097,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4882774353027344,
      "rewards/margins": 0.34271863102912903,
      "rewards/rejected": -1.8309959173202515,
      "sft_loss": 1.5207953453063965,
      "step": 2185
    },
    {
      "epoch": 1.1721023582538885,
      "grad_norm": 5.104389335074668,
      "learning_rate": 7.638933899585354e-07,
      "logits/chosen": 0.041389111429452896,
      "logits/rejected": 0.10888870805501938,
      "logps/chosen": -1.471793293952942,
      "logps/rejected": -1.8024581670761108,
      "loss": 0.5159,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.471793293952942,
      "rewards/margins": 0.33066508173942566,
      "rewards/rejected": -1.8024581670761108,
      "sft_loss": 1.6215784549713135,
      "step": 2190
    },
    {
      "epoch": 1.1747783910352902,
      "grad_norm": 6.218059706938761,
      "learning_rate": 7.625692901064573e-07,
      "logits/chosen": -0.06100373715162277,
      "logits/rejected": 0.03840227052569389,
      "logps/chosen": -1.4952716827392578,
      "logps/rejected": -1.9705145359039307,
      "loss": 0.482,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4952716827392578,
      "rewards/margins": 0.47524294257164,
      "rewards/rejected": -1.9705145359039307,
      "sft_loss": 1.6023286581039429,
      "step": 2195
    },
    {
      "epoch": 1.1774544238166917,
      "grad_norm": 6.667214648831752,
      "learning_rate": 7.61242642805975e-07,
      "logits/chosen": -0.15195073187351227,
      "logits/rejected": -0.15606653690338135,
      "logps/chosen": -1.4973087310791016,
      "logps/rejected": -1.866207480430603,
      "loss": 0.5178,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4973087310791016,
      "rewards/margins": 0.368898868560791,
      "rewards/rejected": -1.866207480430603,
      "sft_loss": 1.6012775897979736,
      "step": 2200
    },
    {
      "epoch": 1.1801304565980932,
      "grad_norm": 4.400604592157586,
      "learning_rate": 7.599134609282266e-07,
      "logits/chosen": -0.17750287055969238,
      "logits/rejected": 0.00775283295661211,
      "logps/chosen": -1.3903168439865112,
      "logps/rejected": -1.8271557092666626,
      "loss": 0.4611,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3903168439865112,
      "rewards/margins": 0.43683862686157227,
      "rewards/rejected": -1.8271557092666626,
      "sft_loss": 1.4089605808258057,
      "step": 2205
    },
    {
      "epoch": 1.182806489379495,
      "grad_norm": 5.489357722594127,
      "learning_rate": 7.585817573689402e-07,
      "logits/chosen": -0.2346876859664917,
      "logits/rejected": -0.10219166427850723,
      "logps/chosen": -1.3173081874847412,
      "logps/rejected": -1.856581687927246,
      "loss": 0.4336,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3173081874847412,
      "rewards/margins": 0.5392736196517944,
      "rewards/rejected": -1.856581687927246,
      "sft_loss": 1.402467966079712,
      "step": 2210
    },
    {
      "epoch": 1.1854825221608964,
      "grad_norm": 8.669951353681238,
      "learning_rate": 7.572475450483098e-07,
      "logits/chosen": -0.18713152408599854,
      "logits/rejected": -0.09777853637933731,
      "logps/chosen": -1.5724120140075684,
      "logps/rejected": -2.0409770011901855,
      "loss": 0.5026,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5724120140075684,
      "rewards/margins": 0.4685649275779724,
      "rewards/rejected": -2.0409770011901855,
      "sft_loss": 1.5465257167816162,
      "step": 2215
    },
    {
      "epoch": 1.188158554942298,
      "grad_norm": 7.3927574477974645,
      "learning_rate": 7.559108369108689e-07,
      "logits/chosen": -0.23059293627738953,
      "logits/rejected": -0.09404326975345612,
      "logps/chosen": -1.4030895233154297,
      "logps/rejected": -1.7947998046875,
      "loss": 0.5086,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4030895233154297,
      "rewards/margins": 0.3917103409767151,
      "rewards/rejected": -1.7947998046875,
      "sft_loss": 1.4927089214324951,
      "step": 2220
    },
    {
      "epoch": 1.1908345877236997,
      "grad_norm": 7.335250225773767,
      "learning_rate": 7.54571645925366e-07,
      "logits/chosen": -0.22742590308189392,
      "logits/rejected": -0.016776692122220993,
      "logps/chosen": -1.3784120082855225,
      "logps/rejected": -1.9083884954452515,
      "loss": 0.4564,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3784120082855225,
      "rewards/margins": 0.5299763679504395,
      "rewards/rejected": -1.9083884954452515,
      "sft_loss": 1.474208950996399,
      "step": 2225
    },
    {
      "epoch": 1.1935106205051011,
      "grad_norm": 8.880089590181422,
      "learning_rate": 7.532299850846378e-07,
      "logits/chosen": -0.20148631930351257,
      "logits/rejected": -0.04945765435695648,
      "logps/chosen": -1.472278356552124,
      "logps/rejected": -2.0694618225097656,
      "loss": 0.4684,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.472278356552124,
      "rewards/margins": 0.5971832871437073,
      "rewards/rejected": -2.0694618225097656,
      "sft_loss": 1.5050463676452637,
      "step": 2230
    },
    {
      "epoch": 1.1961866532865026,
      "grad_norm": 6.805808478371301,
      "learning_rate": 7.518858674054838e-07,
      "logits/chosen": -0.16491694748401642,
      "logits/rejected": 0.033459730446338654,
      "logps/chosen": -1.4134385585784912,
      "logps/rejected": -1.9295330047607422,
      "loss": 0.479,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4134385585784912,
      "rewards/margins": 0.5160945057868958,
      "rewards/rejected": -1.9295330047607422,
      "sft_loss": 1.4623357057571411,
      "step": 2235
    },
    {
      "epoch": 1.1988626860679044,
      "grad_norm": 5.422856370689168,
      "learning_rate": 7.505393059285394e-07,
      "logits/chosen": -0.1376948058605194,
      "logits/rejected": 0.04011141508817673,
      "logps/chosen": -1.4443703889846802,
      "logps/rejected": -1.9348556995391846,
      "loss": 0.4503,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4443703889846802,
      "rewards/margins": 0.490485280752182,
      "rewards/rejected": -1.9348556995391846,
      "sft_loss": 1.5311527252197266,
      "step": 2240
    },
    {
      "epoch": 1.2015387188493059,
      "grad_norm": 6.783491099675574,
      "learning_rate": 7.491903137181501e-07,
      "logits/chosen": -0.10291273891925812,
      "logits/rejected": -0.04401315003633499,
      "logps/chosen": -1.4417774677276611,
      "logps/rejected": -1.8393230438232422,
      "loss": 0.4812,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4417774677276611,
      "rewards/margins": 0.39754557609558105,
      "rewards/rejected": -1.8393230438232422,
      "sft_loss": 1.530505657196045,
      "step": 2245
    },
    {
      "epoch": 1.2042147516307076,
      "grad_norm": 7.2681706027953625,
      "learning_rate": 7.478389038622441e-07,
      "logits/chosen": -0.06046579033136368,
      "logits/rejected": -0.036283593624830246,
      "logps/chosen": -1.4516606330871582,
      "logps/rejected": -1.9776906967163086,
      "loss": 0.4446,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4516606330871582,
      "rewards/margins": 0.5260300636291504,
      "rewards/rejected": -1.9776906967163086,
      "sft_loss": 1.5176465511322021,
      "step": 2250
    },
    {
      "epoch": 1.206890784412109,
      "grad_norm": 8.607018105014358,
      "learning_rate": 7.46485089472206e-07,
      "logits/chosen": -0.13837631046772003,
      "logits/rejected": -0.025722693651914597,
      "logps/chosen": -1.5705254077911377,
      "logps/rejected": -1.8737900257110596,
      "loss": 0.5645,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.5705254077911377,
      "rewards/margins": 0.3032645881175995,
      "rewards/rejected": -1.8737900257110596,
      "sft_loss": 1.5998989343643188,
      "step": 2255
    },
    {
      "epoch": 1.2095668171935106,
      "grad_norm": 6.195684795946535,
      "learning_rate": 7.451288836827487e-07,
      "logits/chosen": -0.08365651220083237,
      "logits/rejected": -0.0891302078962326,
      "logps/chosen": -1.462726354598999,
      "logps/rejected": -1.742959976196289,
      "loss": 0.5299,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.462726354598999,
      "rewards/margins": 0.2802334725856781,
      "rewards/rejected": -1.742959976196289,
      "sft_loss": 1.5141435861587524,
      "step": 2260
    },
    {
      "epoch": 1.2122428499749123,
      "grad_norm": 8.866576424845162,
      "learning_rate": 7.437702996517869e-07,
      "logits/chosen": -0.18240386247634888,
      "logits/rejected": -0.07936345040798187,
      "logps/chosen": -1.5119251012802124,
      "logps/rejected": -1.9127542972564697,
      "loss": 0.5063,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5119251012802124,
      "rewards/margins": 0.400829017162323,
      "rewards/rejected": -1.9127542972564697,
      "sft_loss": 1.6090915203094482,
      "step": 2265
    },
    {
      "epoch": 1.2149188827563138,
      "grad_norm": 9.73327657484205,
      "learning_rate": 7.424093505603087e-07,
      "logits/chosen": -0.26718050241470337,
      "logits/rejected": -0.0880444124341011,
      "logps/chosen": -1.5113370418548584,
      "logps/rejected": -2.0135390758514404,
      "loss": 0.491,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5113370418548584,
      "rewards/margins": 0.5022020936012268,
      "rewards/rejected": -2.0135390758514404,
      "sft_loss": 1.5244777202606201,
      "step": 2270
    },
    {
      "epoch": 1.2175949155377153,
      "grad_norm": 7.068729022101841,
      "learning_rate": 7.410460496122482e-07,
      "logits/chosen": -0.14537358283996582,
      "logits/rejected": -0.008987700566649437,
      "logps/chosen": -1.4155831336975098,
      "logps/rejected": -1.9859249591827393,
      "loss": 0.4504,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4155831336975098,
      "rewards/margins": 0.5703418850898743,
      "rewards/rejected": -1.9859249591827393,
      "sft_loss": 1.4823784828186035,
      "step": 2275
    },
    {
      "epoch": 1.220270948319117,
      "grad_norm": 10.14119788003141,
      "learning_rate": 7.396804100343572e-07,
      "logits/chosen": -0.20737072825431824,
      "logits/rejected": -0.033312804996967316,
      "logps/chosen": -1.3519184589385986,
      "logps/rejected": -1.7951476573944092,
      "loss": 0.4764,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3519184589385986,
      "rewards/margins": 0.44322913885116577,
      "rewards/rejected": -1.7951476573944092,
      "sft_loss": 1.4460406303405762,
      "step": 2280
    },
    {
      "epoch": 1.2229469811005185,
      "grad_norm": 5.044558530057508,
      "learning_rate": 7.383124450760768e-07,
      "logits/chosen": -0.13140533864498138,
      "logits/rejected": 0.07922891527414322,
      "logps/chosen": -1.5612608194351196,
      "logps/rejected": -2.098231315612793,
      "loss": 0.4731,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5612608194351196,
      "rewards/margins": 0.5369703769683838,
      "rewards/rejected": -2.098231315612793,
      "sft_loss": 1.5844147205352783,
      "step": 2285
    },
    {
      "epoch": 1.22562301388192,
      "grad_norm": 5.122452335919795,
      "learning_rate": 7.369421680094091e-07,
      "logits/chosen": -0.2452307939529419,
      "logits/rejected": -0.07779756188392639,
      "logps/chosen": -1.4014132022857666,
      "logps/rejected": -1.8258777856826782,
      "loss": 0.4941,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4014132022857666,
      "rewards/margins": 0.4244648516178131,
      "rewards/rejected": -1.8258777856826782,
      "sft_loss": 1.4864047765731812,
      "step": 2290
    },
    {
      "epoch": 1.2282990466633217,
      "grad_norm": 6.735504262827155,
      "learning_rate": 7.355695921287881e-07,
      "logits/chosen": -0.1678198277950287,
      "logits/rejected": -0.07633207738399506,
      "logps/chosen": -1.4897345304489136,
      "logps/rejected": -1.9370830059051514,
      "loss": 0.5134,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4897345304489136,
      "rewards/margins": 0.4473484456539154,
      "rewards/rejected": -1.9370830059051514,
      "sft_loss": 1.6406726837158203,
      "step": 2295
    },
    {
      "epoch": 1.2309750794447232,
      "grad_norm": 9.594350265687709,
      "learning_rate": 7.341947307509513e-07,
      "logits/chosen": -0.12510626018047333,
      "logits/rejected": 0.01086263358592987,
      "logps/chosen": -1.452544927597046,
      "logps/rejected": -1.848858118057251,
      "loss": 0.5141,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.452544927597046,
      "rewards/margins": 0.39631298184394836,
      "rewards/rejected": -1.848858118057251,
      "sft_loss": 1.5406442880630493,
      "step": 2300
    },
    {
      "epoch": 1.233651112226125,
      "grad_norm": 7.56576661024348,
      "learning_rate": 7.328175972148094e-07,
      "logits/chosen": -0.15516726672649384,
      "logits/rejected": -0.007723456714302301,
      "logps/chosen": -1.6164741516113281,
      "logps/rejected": -2.134988307952881,
      "loss": 0.4919,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6164741516113281,
      "rewards/margins": 0.5185142755508423,
      "rewards/rejected": -2.134988307952881,
      "sft_loss": 1.5903704166412354,
      "step": 2305
    },
    {
      "epoch": 1.2363271450075264,
      "grad_norm": 7.056473066838253,
      "learning_rate": 7.314382048813185e-07,
      "logits/chosen": -0.10760338604450226,
      "logits/rejected": 0.16948586702346802,
      "logps/chosen": -1.4854151010513306,
      "logps/rejected": -2.0326244831085205,
      "loss": 0.477,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4854151010513306,
      "rewards/margins": 0.5472094416618347,
      "rewards/rejected": -2.0326244831085205,
      "sft_loss": 1.5109024047851562,
      "step": 2310
    },
    {
      "epoch": 1.2390031777889279,
      "grad_norm": 6.750522317956841,
      "learning_rate": 7.300565671333486e-07,
      "logits/chosen": -0.12459911406040192,
      "logits/rejected": 0.06452270597219467,
      "logps/chosen": -1.5161750316619873,
      "logps/rejected": -2.0566658973693848,
      "loss": 0.4867,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.5161750316619873,
      "rewards/margins": 0.5404911637306213,
      "rewards/rejected": -2.0566658973693848,
      "sft_loss": 1.5886337757110596,
      "step": 2315
    },
    {
      "epoch": 1.2416792105703296,
      "grad_norm": 5.275621529765781,
      "learning_rate": 7.286726973755554e-07,
      "logits/chosen": -0.014655408449470997,
      "logits/rejected": 0.020419131964445114,
      "logps/chosen": -1.4924757480621338,
      "logps/rejected": -1.9487247467041016,
      "loss": 0.4934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4924757480621338,
      "rewards/margins": 0.4562491476535797,
      "rewards/rejected": -1.9487247467041016,
      "sft_loss": 1.5474860668182373,
      "step": 2320
    },
    {
      "epoch": 1.244355243351731,
      "grad_norm": 6.810408731308987,
      "learning_rate": 7.272866090342493e-07,
      "logits/chosen": 0.025302385911345482,
      "logits/rejected": 0.11866162717342377,
      "logps/chosen": -1.5684155225753784,
      "logps/rejected": -2.068596601486206,
      "loss": 0.4883,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5684155225753784,
      "rewards/margins": 0.5001809000968933,
      "rewards/rejected": -2.068596601486206,
      "sft_loss": 1.557191014289856,
      "step": 2325
    },
    {
      "epoch": 1.2470312761331326,
      "grad_norm": 6.350633612711436,
      "learning_rate": 7.258983155572656e-07,
      "logits/chosen": -0.1877928078174591,
      "logits/rejected": -0.06601401418447495,
      "logps/chosen": -1.4745948314666748,
      "logps/rejected": -1.8821157217025757,
      "loss": 0.5188,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.4745948314666748,
      "rewards/margins": 0.4075208306312561,
      "rewards/rejected": -1.8821157217025757,
      "sft_loss": 1.5776586532592773,
      "step": 2330
    },
    {
      "epoch": 1.2497073089145343,
      "grad_norm": 5.982336730467969,
      "learning_rate": 7.245078304138335e-07,
      "logits/chosen": -0.020032238215208054,
      "logits/rejected": 0.052525751292705536,
      "logps/chosen": -1.539282202720642,
      "logps/rejected": -2.0581915378570557,
      "loss": 0.4677,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.539282202720642,
      "rewards/margins": 0.5189090967178345,
      "rewards/rejected": -2.0581915378570557,
      "sft_loss": 1.6047948598861694,
      "step": 2335
    },
    {
      "epoch": 1.2523833416959358,
      "grad_norm": 5.214865914779404,
      "learning_rate": 7.231151670944462e-07,
      "logits/chosen": -0.19301816821098328,
      "logits/rejected": 0.0031130120623856783,
      "logps/chosen": -1.5542106628417969,
      "logps/rejected": -1.9580436944961548,
      "loss": 0.5151,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5542106628417969,
      "rewards/margins": 0.40383315086364746,
      "rewards/rejected": -1.9580436944961548,
      "sft_loss": 1.6011720895767212,
      "step": 2340
    },
    {
      "epoch": 1.2550593744773373,
      "grad_norm": 6.088334407434233,
      "learning_rate": 7.217203391107291e-07,
      "logits/chosen": -0.11648712307214737,
      "logits/rejected": 0.06812908500432968,
      "logps/chosen": -1.4927353858947754,
      "logps/rejected": -1.9419381618499756,
      "loss": 0.4951,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4927353858947754,
      "rewards/margins": 0.4492027163505554,
      "rewards/rejected": -1.9419381618499756,
      "sft_loss": 1.5813872814178467,
      "step": 2345
    },
    {
      "epoch": 1.257735407258739,
      "grad_norm": 6.118015819779254,
      "learning_rate": 7.203233599953096e-07,
      "logits/chosen": -0.12596911191940308,
      "logits/rejected": 0.03608747571706772,
      "logps/chosen": -1.5252940654754639,
      "logps/rejected": -1.930368185043335,
      "loss": 0.5163,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5252940654754639,
      "rewards/margins": 0.4050741195678711,
      "rewards/rejected": -1.930368185043335,
      "sft_loss": 1.5665839910507202,
      "step": 2350
    },
    {
      "epoch": 1.2604114400401405,
      "grad_norm": 7.019082824371375,
      "learning_rate": 7.189242433016852e-07,
      "logits/chosen": -0.04565399885177612,
      "logits/rejected": 0.10724995285272598,
      "logps/chosen": -1.3797297477722168,
      "logps/rejected": -1.9839117527008057,
      "loss": 0.4677,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.3797297477722168,
      "rewards/margins": 0.6041820049285889,
      "rewards/rejected": -1.9839117527008057,
      "sft_loss": 1.4846323728561401,
      "step": 2355
    },
    {
      "epoch": 1.263087472821542,
      "grad_norm": 9.39532993132623,
      "learning_rate": 7.17523002604092e-07,
      "logits/chosen": -0.06291689723730087,
      "logits/rejected": 0.09211830794811249,
      "logps/chosen": -1.4899945259094238,
      "logps/rejected": -2.1186699867248535,
      "loss": 0.4564,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4899945259094238,
      "rewards/margins": 0.6286753416061401,
      "rewards/rejected": -2.1186699867248535,
      "sft_loss": 1.5955866575241089,
      "step": 2360
    },
    {
      "epoch": 1.2657635056029437,
      "grad_norm": 4.337083396947069,
      "learning_rate": 7.161196514973734e-07,
      "logits/chosen": -0.06913149356842041,
      "logits/rejected": 0.07702628523111343,
      "logps/chosen": -1.4909687042236328,
      "logps/rejected": -2.0008749961853027,
      "loss": 0.488,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4909687042236328,
      "rewards/margins": 0.5099064111709595,
      "rewards/rejected": -2.0008749961853027,
      "sft_loss": 1.5813932418823242,
      "step": 2365
    },
    {
      "epoch": 1.2684395383843452,
      "grad_norm": 9.719493267673636,
      "learning_rate": 7.147142035968483e-07,
      "logits/chosen": -0.057274866849184036,
      "logits/rejected": 0.08096132427453995,
      "logps/chosen": -1.5333844423294067,
      "logps/rejected": -1.9823888540267944,
      "loss": 0.4794,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5333844423294067,
      "rewards/margins": 0.44900450110435486,
      "rewards/rejected": -1.9823888540267944,
      "sft_loss": 1.634751319885254,
      "step": 2370
    },
    {
      "epoch": 1.2711155711657467,
      "grad_norm": 6.187677361284805,
      "learning_rate": 7.133066725381781e-07,
      "logits/chosen": -0.1976046860218048,
      "logits/rejected": -0.0066480934619903564,
      "logps/chosen": -1.388067364692688,
      "logps/rejected": -1.7951103448867798,
      "loss": 0.4872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.388067364692688,
      "rewards/margins": 0.40704289078712463,
      "rewards/rejected": -1.7951103448867798,
      "sft_loss": 1.4659048318862915,
      "step": 2375
    },
    {
      "epoch": 1.2737916039471484,
      "grad_norm": 10.546333717242193,
      "learning_rate": 7.118970719772354e-07,
      "logits/chosen": -0.17652028799057007,
      "logits/rejected": 0.04019765183329582,
      "logps/chosen": -1.5461647510528564,
      "logps/rejected": -2.09086275100708,
      "loss": 0.4972,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5461647510528564,
      "rewards/margins": 0.5446978807449341,
      "rewards/rejected": -2.09086275100708,
      "sft_loss": 1.6310644149780273,
      "step": 2380
    },
    {
      "epoch": 1.27646763672855,
      "grad_norm": 5.631643384912899,
      "learning_rate": 7.104854155899711e-07,
      "logits/chosen": -0.017530517652630806,
      "logits/rejected": 0.10928189754486084,
      "logps/chosen": -1.5052597522735596,
      "logps/rejected": -1.9419084787368774,
      "loss": 0.5042,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5052597522735596,
      "rewards/margins": 0.4366486072540283,
      "rewards/rejected": -1.9419084787368774,
      "sft_loss": 1.5173150300979614,
      "step": 2385
    },
    {
      "epoch": 1.2791436695099514,
      "grad_norm": 5.330658404953745,
      "learning_rate": 7.090717170722817e-07,
      "logits/chosen": -0.040447741746902466,
      "logits/rejected": 0.03319546580314636,
      "logps/chosen": -1.4307003021240234,
      "logps/rejected": -2.0200963020324707,
      "loss": 0.4367,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4307003021240234,
      "rewards/margins": 0.5893961191177368,
      "rewards/rejected": -2.0200963020324707,
      "sft_loss": 1.532576560974121,
      "step": 2390
    },
    {
      "epoch": 1.2818197022913531,
      "grad_norm": 6.503741181473453,
      "learning_rate": 7.076559901398762e-07,
      "logits/chosen": -0.22894029319286346,
      "logits/rejected": -0.08977676928043365,
      "logps/chosen": -1.39363694190979,
      "logps/rejected": -1.8419349193572998,
      "loss": 0.4658,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.39363694190979,
      "rewards/margins": 0.44829821586608887,
      "rewards/rejected": -1.8419349193572998,
      "sft_loss": 1.4875110387802124,
      "step": 2395
    },
    {
      "epoch": 1.2844957350727546,
      "grad_norm": 8.559354194639448,
      "learning_rate": 7.062382485281436e-07,
      "logits/chosen": -0.08281628787517548,
      "logits/rejected": 0.05056994408369064,
      "logps/chosen": -1.4323084354400635,
      "logps/rejected": -1.8499174118041992,
      "loss": 0.4824,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4323084354400635,
      "rewards/margins": 0.41760897636413574,
      "rewards/rejected": -1.8499174118041992,
      "sft_loss": 1.5048401355743408,
      "step": 2400
    },
    {
      "epoch": 1.2844957350727546,
      "eval_logits/chosen": 0.21058468520641327,
      "eval_logits/rejected": 0.3098817765712738,
      "eval_logps/chosen": -1.5341204404830933,
      "eval_logps/rejected": -2.0179104804992676,
      "eval_loss": 0.49633318185806274,
      "eval_rewards/accuracies": 0.6535608172416687,
      "eval_rewards/chosen": -1.5341204404830933,
      "eval_rewards/margins": 0.48379021883010864,
      "eval_rewards/rejected": -2.0179104804992676,
      "eval_runtime": 43.3901,
      "eval_samples_per_second": 30.998,
      "eval_sft_loss": 1.6007072925567627,
      "eval_steps_per_second": 7.767,
      "step": 2400
    },
    {
      "epoch": 1.287171767854156,
      "grad_norm": 4.898881519182048,
      "learning_rate": 7.048185059920193e-07,
      "logits/chosen": -0.06763359159231186,
      "logits/rejected": 0.10140136629343033,
      "logps/chosen": -1.5375685691833496,
      "logps/rejected": -2.103701591491699,
      "loss": 0.5041,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5375685691833496,
      "rewards/margins": 0.5661331415176392,
      "rewards/rejected": -2.103701591491699,
      "sft_loss": 1.5908316373825073,
      "step": 2405
    },
    {
      "epoch": 1.2898478006355578,
      "grad_norm": 7.678119509091904,
      "learning_rate": 7.033967763058516e-07,
      "logits/chosen": -0.20165202021598816,
      "logits/rejected": 0.02417032979428768,
      "logps/chosen": -1.4833492040634155,
      "logps/rejected": -1.7696387767791748,
      "loss": 0.5147,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4833492040634155,
      "rewards/margins": 0.28628966212272644,
      "rewards/rejected": -1.7696387767791748,
      "sft_loss": 1.563765525817871,
      "step": 2410
    },
    {
      "epoch": 1.2925238334169593,
      "grad_norm": 5.865490300549153,
      "learning_rate": 7.019730732632681e-07,
      "logits/chosen": -0.04911652207374573,
      "logits/rejected": 0.03724956512451172,
      "logps/chosen": -1.4184011220932007,
      "logps/rejected": -2.0442159175872803,
      "loss": 0.4609,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4184011220932007,
      "rewards/margins": 0.6258147954940796,
      "rewards/rejected": -2.0442159175872803,
      "sft_loss": 1.4617655277252197,
      "step": 2415
    },
    {
      "epoch": 1.2951998661983608,
      "grad_norm": 5.003988637593825,
      "learning_rate": 7.005474106770418e-07,
      "logits/chosen": -0.1761176586151123,
      "logits/rejected": -0.02824026718735695,
      "logps/chosen": -1.5366970300674438,
      "logps/rejected": -2.046592950820923,
      "loss": 0.4524,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5366970300674438,
      "rewards/margins": 0.509895920753479,
      "rewards/rejected": -2.046592950820923,
      "sft_loss": 1.6355535984039307,
      "step": 2420
    },
    {
      "epoch": 1.2978758989797625,
      "grad_norm": 5.068686232645391,
      "learning_rate": 6.991198023789577e-07,
      "logits/chosen": -0.064757339656353,
      "logits/rejected": 0.01880965568125248,
      "logps/chosen": -1.406187891960144,
      "logps/rejected": -1.8038914203643799,
      "loss": 0.4605,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.406187891960144,
      "rewards/margins": 0.3977034091949463,
      "rewards/rejected": -1.8038914203643799,
      "sft_loss": 1.528364896774292,
      "step": 2425
    },
    {
      "epoch": 1.300551931761164,
      "grad_norm": 15.350732764917854,
      "learning_rate": 6.976902622196776e-07,
      "logits/chosen": -0.05523641034960747,
      "logits/rejected": 0.01888253539800644,
      "logps/chosen": -1.5876901149749756,
      "logps/rejected": -2.023746967315674,
      "loss": 0.5078,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5876901149749756,
      "rewards/margins": 0.436056911945343,
      "rewards/rejected": -2.023746967315674,
      "sft_loss": 1.6234697103500366,
      "step": 2430
    },
    {
      "epoch": 1.3032279645425655,
      "grad_norm": 6.027884750611186,
      "learning_rate": 6.962588040686064e-07,
      "logits/chosen": -0.04204241558909416,
      "logits/rejected": 0.09753796458244324,
      "logps/chosen": -1.4533779621124268,
      "logps/rejected": -1.7768032550811768,
      "loss": 0.5119,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4533779621124268,
      "rewards/margins": 0.3234253525733948,
      "rewards/rejected": -1.7768032550811768,
      "sft_loss": 1.493114948272705,
      "step": 2435
    },
    {
      "epoch": 1.3059039973239672,
      "grad_norm": 7.734356836200579,
      "learning_rate": 6.948254418137573e-07,
      "logits/chosen": -0.1577232927083969,
      "logits/rejected": -0.027221685275435448,
      "logps/chosen": -1.4389315843582153,
      "logps/rejected": -1.936092734336853,
      "loss": 0.4825,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4389315843582153,
      "rewards/margins": 0.49716120958328247,
      "rewards/rejected": -1.936092734336853,
      "sft_loss": 1.449273943901062,
      "step": 2440
    },
    {
      "epoch": 1.3085800301053687,
      "grad_norm": 7.371662500469119,
      "learning_rate": 6.933901893616174e-07,
      "logits/chosen": -0.10918200016021729,
      "logits/rejected": 0.04994308948516846,
      "logps/chosen": -1.4908276796340942,
      "logps/rejected": -1.8749077320098877,
      "loss": 0.5134,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4908276796340942,
      "rewards/margins": 0.38408005237579346,
      "rewards/rejected": -1.8749077320098877,
      "sft_loss": 1.526848554611206,
      "step": 2445
    },
    {
      "epoch": 1.3112560628867704,
      "grad_norm": 7.61863988294174,
      "learning_rate": 6.919530606370121e-07,
      "logits/chosen": -0.09959923475980759,
      "logits/rejected": 0.06935176998376846,
      "logps/chosen": -1.423628807067871,
      "logps/rejected": -1.949601411819458,
      "loss": 0.4824,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.423628807067871,
      "rewards/margins": 0.5259725451469421,
      "rewards/rejected": -1.949601411819458,
      "sft_loss": 1.4399412870407104,
      "step": 2450
    },
    {
      "epoch": 1.313932095668172,
      "grad_norm": 4.815212043827955,
      "learning_rate": 6.905140695829706e-07,
      "logits/chosen": -0.1309075951576233,
      "logits/rejected": 0.1330709159374237,
      "logps/chosen": -1.513875126838684,
      "logps/rejected": -1.957093596458435,
      "loss": 0.4915,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.513875126838684,
      "rewards/margins": 0.44321855902671814,
      "rewards/rejected": -1.957093596458435,
      "sft_loss": 1.5696543455123901,
      "step": 2455
    },
    {
      "epoch": 1.3166081284495736,
      "grad_norm": 11.562215211111699,
      "learning_rate": 6.890732301605904e-07,
      "logits/chosen": -0.060210179537534714,
      "logits/rejected": 0.06136656925082207,
      "logps/chosen": -1.4934711456298828,
      "logps/rejected": -1.8130403757095337,
      "loss": 0.5434,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4934711456298828,
      "rewards/margins": 0.31956934928894043,
      "rewards/rejected": -1.8130403757095337,
      "sft_loss": 1.5263965129852295,
      "step": 2460
    },
    {
      "epoch": 1.3192841612309751,
      "grad_norm": 5.522447694900418,
      "learning_rate": 6.876305563489021e-07,
      "logits/chosen": -0.06598939001560211,
      "logits/rejected": 0.02520240843296051,
      "logps/chosen": -1.4613018035888672,
      "logps/rejected": -2.028432846069336,
      "loss": 0.4699,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4613018035888672,
      "rewards/margins": 0.5671309232711792,
      "rewards/rejected": -2.028432846069336,
      "sft_loss": 1.480936884880066,
      "step": 2465
    },
    {
      "epoch": 1.3219601940123766,
      "grad_norm": 6.621376942660697,
      "learning_rate": 6.861860621447331e-07,
      "logits/chosen": -0.2126634120941162,
      "logits/rejected": -0.06398321688175201,
      "logps/chosen": -1.438204288482666,
      "logps/rejected": -1.7354466915130615,
      "loss": 0.5138,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.438204288482666,
      "rewards/margins": 0.2972424626350403,
      "rewards/rejected": -1.7354466915130615,
      "sft_loss": 1.5432885885238647,
      "step": 2470
    },
    {
      "epoch": 1.3246362267937783,
      "grad_norm": 5.988214386261141,
      "learning_rate": 6.847397615625725e-07,
      "logits/chosen": -0.07591654360294342,
      "logits/rejected": -7.105171971488744e-05,
      "logps/chosen": -1.4517724514007568,
      "logps/rejected": -1.8585712909698486,
      "loss": 0.4723,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4517724514007568,
      "rewards/margins": 0.40679866075515747,
      "rewards/rejected": -1.8585712909698486,
      "sft_loss": 1.5348371267318726,
      "step": 2475
    },
    {
      "epoch": 1.3273122595751798,
      "grad_norm": 6.05179775803104,
      "learning_rate": 6.83291668634435e-07,
      "logits/chosen": -0.22602811455726624,
      "logits/rejected": -0.03507426753640175,
      "logps/chosen": -1.504457712173462,
      "logps/rejected": -2.0625691413879395,
      "loss": 0.4524,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.504457712173462,
      "rewards/margins": 0.5581115484237671,
      "rewards/rejected": -2.0625691413879395,
      "sft_loss": 1.6593469381332397,
      "step": 2480
    },
    {
      "epoch": 1.3299882923565813,
      "grad_norm": 5.612677415801948,
      "learning_rate": 6.818417974097246e-07,
      "logits/chosen": -0.04492194578051567,
      "logits/rejected": 0.1510792225599289,
      "logps/chosen": -1.5203931331634521,
      "logps/rejected": -2.1091973781585693,
      "loss": 0.505,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5203931331634521,
      "rewards/margins": 0.588804304599762,
      "rewards/rejected": -2.1091973781585693,
      "sft_loss": 1.6557775735855103,
      "step": 2485
    },
    {
      "epoch": 1.332664325137983,
      "grad_norm": 8.054763097158059,
      "learning_rate": 6.803901619550981e-07,
      "logits/chosen": -0.1551412045955658,
      "logits/rejected": -0.09231224656105042,
      "logps/chosen": -1.55299973487854,
      "logps/rejected": -2.0047755241394043,
      "loss": 0.4886,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.55299973487854,
      "rewards/margins": 0.45177555084228516,
      "rewards/rejected": -2.0047755241394043,
      "sft_loss": 1.62527596950531,
      "step": 2490
    },
    {
      "epoch": 1.3353403579193845,
      "grad_norm": 6.649605258874318,
      "learning_rate": 6.789367763543292e-07,
      "logits/chosen": -0.03161294385790825,
      "logits/rejected": -0.020646898075938225,
      "logps/chosen": -1.4861562252044678,
      "logps/rejected": -1.8771871328353882,
      "loss": 0.5127,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4861562252044678,
      "rewards/margins": 0.39103081822395325,
      "rewards/rejected": -1.8771871328353882,
      "sft_loss": 1.543869137763977,
      "step": 2495
    },
    {
      "epoch": 1.338016390700786,
      "grad_norm": 5.826374531410804,
      "learning_rate": 6.774816547081714e-07,
      "logits/chosen": -0.06338343769311905,
      "logits/rejected": 0.1107725128531456,
      "logps/chosen": -1.4578975439071655,
      "logps/rejected": -1.8605518341064453,
      "loss": 0.4885,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4578975439071655,
      "rewards/margins": 0.402654230594635,
      "rewards/rejected": -1.8605518341064453,
      "sft_loss": 1.57229483127594,
      "step": 2500
    },
    {
      "epoch": 1.3406924234821878,
      "grad_norm": 6.41371973658278,
      "learning_rate": 6.760248111342211e-07,
      "logits/chosen": -0.04192394018173218,
      "logits/rejected": 0.14050684869289398,
      "logps/chosen": -1.3707352876663208,
      "logps/rejected": -1.8369287252426147,
      "loss": 0.4817,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3707352876663208,
      "rewards/margins": 0.4661934971809387,
      "rewards/rejected": -1.8369287252426147,
      "sft_loss": 1.4329276084899902,
      "step": 2505
    },
    {
      "epoch": 1.3433684562635893,
      "grad_norm": 6.924270674766152,
      "learning_rate": 6.745662597667813e-07,
      "logits/chosen": -0.1159440129995346,
      "logits/rejected": 0.03930569440126419,
      "logps/chosen": -1.4073395729064941,
      "logps/rejected": -1.8557002544403076,
      "loss": 0.4794,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4073395729064941,
      "rewards/margins": 0.44836077094078064,
      "rewards/rejected": -1.8557002544403076,
      "sft_loss": 1.4969781637191772,
      "step": 2510
    },
    {
      "epoch": 1.3460444890449907,
      "grad_norm": 6.41722619481814,
      "learning_rate": 6.731060147567236e-07,
      "logits/chosen": -0.047092996537685394,
      "logits/rejected": 0.08643031120300293,
      "logps/chosen": -1.4872676134109497,
      "logps/rejected": -1.9126088619232178,
      "loss": 0.4952,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4872676134109497,
      "rewards/margins": 0.4253414273262024,
      "rewards/rejected": -1.9126088619232178,
      "sft_loss": 1.61127507686615,
      "step": 2515
    },
    {
      "epoch": 1.3487205218263925,
      "grad_norm": 7.916966014649849,
      "learning_rate": 6.716440902713515e-07,
      "logits/chosen": -0.1546896994113922,
      "logits/rejected": -0.06426952034235,
      "logps/chosen": -1.4904658794403076,
      "logps/rejected": -1.8696438074111938,
      "loss": 0.4876,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4904658794403076,
      "rewards/margins": 0.3791780471801758,
      "rewards/rejected": -1.8696438074111938,
      "sft_loss": 1.4804034233093262,
      "step": 2520
    },
    {
      "epoch": 1.351396554607794,
      "grad_norm": 8.077287376418578,
      "learning_rate": 6.701805004942627e-07,
      "logits/chosen": -0.10523150116205215,
      "logits/rejected": -0.01580960303544998,
      "logps/chosen": -1.5716774463653564,
      "logps/rejected": -2.037581205368042,
      "loss": 0.4826,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5716774463653564,
      "rewards/margins": 0.4659039080142975,
      "rewards/rejected": -2.037581205368042,
      "sft_loss": 1.7261673212051392,
      "step": 2525
    },
    {
      "epoch": 1.3540725873891954,
      "grad_norm": 10.231820821782932,
      "learning_rate": 6.687152596252119e-07,
      "logits/chosen": -0.1276179999113083,
      "logits/rejected": -0.06458455324172974,
      "logps/chosen": -1.5572798252105713,
      "logps/rejected": -1.858924150466919,
      "loss": 0.5626,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.5572798252105713,
      "rewards/margins": 0.3016444146633148,
      "rewards/rejected": -1.858924150466919,
      "sft_loss": 1.6395533084869385,
      "step": 2530
    },
    {
      "epoch": 1.3567486201705972,
      "grad_norm": 4.578023625051732,
      "learning_rate": 6.672483818799722e-07,
      "logits/chosen": -0.18592733144760132,
      "logits/rejected": -0.018612224608659744,
      "logps/chosen": -1.448899269104004,
      "logps/rejected": -1.9447243213653564,
      "loss": 0.4658,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.448899269104004,
      "rewards/margins": 0.49582499265670776,
      "rewards/rejected": -1.9447243213653564,
      "sft_loss": 1.5233696699142456,
      "step": 2535
    },
    {
      "epoch": 1.3594246529519987,
      "grad_norm": 7.529452510480847,
      "learning_rate": 6.657798814901978e-07,
      "logits/chosen": -0.1055329218506813,
      "logits/rejected": 0.09222625941038132,
      "logps/chosen": -1.5448589324951172,
      "logps/rejected": -1.9362952709197998,
      "loss": 0.514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5448589324951172,
      "rewards/margins": 0.39143624901771545,
      "rewards/rejected": -1.9362952709197998,
      "sft_loss": 1.6498651504516602,
      "step": 2540
    },
    {
      "epoch": 1.3621006857334002,
      "grad_norm": 5.551868702561516,
      "learning_rate": 6.643097727032863e-07,
      "logits/chosen": -0.08056094497442245,
      "logits/rejected": 0.1106506809592247,
      "logps/chosen": -1.4577360153198242,
      "logps/rejected": -1.9888372421264648,
      "loss": 0.4764,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4577360153198242,
      "rewards/margins": 0.5311012268066406,
      "rewards/rejected": -1.9888372421264648,
      "sft_loss": 1.5364185571670532,
      "step": 2545
    },
    {
      "epoch": 1.3647767185148019,
      "grad_norm": 5.945591994335276,
      "learning_rate": 6.628380697822392e-07,
      "logits/chosen": -0.11514048278331757,
      "logits/rejected": 0.06404171884059906,
      "logps/chosen": -1.470406413078308,
      "logps/rejected": -1.8387157917022705,
      "loss": 0.501,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.470406413078308,
      "rewards/margins": 0.36830934882164,
      "rewards/rejected": -1.8387157917022705,
      "sft_loss": 1.5712687969207764,
      "step": 2550
    },
    {
      "epoch": 1.3674527512962034,
      "grad_norm": 12.049719645699561,
      "learning_rate": 6.61364787005525e-07,
      "logits/chosen": -0.05989614874124527,
      "logits/rejected": 0.039876788854599,
      "logps/chosen": -1.381488561630249,
      "logps/rejected": -1.919499158859253,
      "loss": 0.4806,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.381488561630249,
      "rewards/margins": 0.5380104780197144,
      "rewards/rejected": -1.919499158859253,
      "sft_loss": 1.464348316192627,
      "step": 2555
    },
    {
      "epoch": 1.3701287840776049,
      "grad_norm": 10.506638498291947,
      "learning_rate": 6.598899386669395e-07,
      "logits/chosen": -0.06798264384269714,
      "logits/rejected": 0.07144947350025177,
      "logps/chosen": -1.478407621383667,
      "logps/rejected": -1.9259250164031982,
      "loss": 0.4827,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.478407621383667,
      "rewards/margins": 0.4475174844264984,
      "rewards/rejected": -1.9259250164031982,
      "sft_loss": 1.546134352684021,
      "step": 2560
    },
    {
      "epoch": 1.3728048168590066,
      "grad_norm": 7.133123060324439,
      "learning_rate": 6.584135390754679e-07,
      "logits/chosen": -0.06579507887363434,
      "logits/rejected": 0.0746840387582779,
      "logps/chosen": -1.4398367404937744,
      "logps/rejected": -2.0183796882629395,
      "loss": 0.446,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4398367404937744,
      "rewards/margins": 0.578542947769165,
      "rewards/rejected": -2.0183796882629395,
      "sft_loss": 1.5617057085037231,
      "step": 2565
    },
    {
      "epoch": 1.375480849640408,
      "grad_norm": 5.902788356834458,
      "learning_rate": 6.569356025551454e-07,
      "logits/chosen": -0.0012567430967465043,
      "logits/rejected": 0.07987087965011597,
      "logps/chosen": -1.449086308479309,
      "logps/rejected": -1.943660020828247,
      "loss": 0.4738,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.449086308479309,
      "rewards/margins": 0.49457383155822754,
      "rewards/rejected": -1.943660020828247,
      "sft_loss": 1.5337722301483154,
      "step": 2570
    },
    {
      "epoch": 1.3781568824218096,
      "grad_norm": 6.881337573413881,
      "learning_rate": 6.554561434449186e-07,
      "logits/chosen": -0.15774737298488617,
      "logits/rejected": 0.0003899022995028645,
      "logps/chosen": -1.416227102279663,
      "logps/rejected": -1.8630616664886475,
      "loss": 0.4742,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.416227102279663,
      "rewards/margins": 0.4468347132205963,
      "rewards/rejected": -1.8630616664886475,
      "sft_loss": 1.4903398752212524,
      "step": 2575
    },
    {
      "epoch": 1.3808329152032113,
      "grad_norm": 5.919829446263836,
      "learning_rate": 6.539751760985063e-07,
      "logits/chosen": -0.0845089852809906,
      "logits/rejected": 0.014165307395160198,
      "logps/chosen": -1.5333517789840698,
      "logps/rejected": -1.879889726638794,
      "loss": 0.5051,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5333517789840698,
      "rewards/margins": 0.34653788805007935,
      "rewards/rejected": -1.879889726638794,
      "sft_loss": 1.6552213430404663,
      "step": 2580
    },
    {
      "epoch": 1.3835089479846128,
      "grad_norm": 6.967963627355345,
      "learning_rate": 6.524927148842602e-07,
      "logits/chosen": 0.04399329423904419,
      "logits/rejected": 0.21903137862682343,
      "logps/chosen": -1.3956737518310547,
      "logps/rejected": -1.9111979007720947,
      "loss": 0.4336,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.3956737518310547,
      "rewards/margins": 0.5155242085456848,
      "rewards/rejected": -1.9111979007720947,
      "sft_loss": 1.4362618923187256,
      "step": 2585
    },
    {
      "epoch": 1.3861849807660143,
      "grad_norm": 7.943512130584483,
      "learning_rate": 6.510087741850254e-07,
      "logits/chosen": -0.08490066230297089,
      "logits/rejected": 0.0671260803937912,
      "logps/chosen": -1.4574105739593506,
      "logps/rejected": -1.8720905780792236,
      "loss": 0.4934,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4574105739593506,
      "rewards/margins": 0.4146800935268402,
      "rewards/rejected": -1.8720905780792236,
      "sft_loss": 1.611249327659607,
      "step": 2590
    },
    {
      "epoch": 1.388861013547416,
      "grad_norm": 5.7395767064625085,
      "learning_rate": 6.495233683980012e-07,
      "logits/chosen": -0.0439167320728302,
      "logits/rejected": 0.013581424951553345,
      "logps/chosen": -1.4777257442474365,
      "logps/rejected": -1.838610053062439,
      "loss": 0.5125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4777257442474365,
      "rewards/margins": 0.36088424921035767,
      "rewards/rejected": -1.838610053062439,
      "sft_loss": 1.5101115703582764,
      "step": 2595
    },
    {
      "epoch": 1.3915370463288175,
      "grad_norm": 6.597324797319752,
      "learning_rate": 6.480365119346011e-07,
      "logits/chosen": 0.04372371733188629,
      "logits/rejected": 0.19077324867248535,
      "logps/chosen": -1.4668786525726318,
      "logps/rejected": -1.8124185800552368,
      "loss": 0.507,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4668786525726318,
      "rewards/margins": 0.34553974866867065,
      "rewards/rejected": -1.8124185800552368,
      "sft_loss": 1.5253021717071533,
      "step": 2600
    },
    {
      "epoch": 1.394213079110219,
      "grad_norm": 10.765397785323103,
      "learning_rate": 6.465482192203129e-07,
      "logits/chosen": 0.017538107931613922,
      "logits/rejected": 0.07946206629276276,
      "logps/chosen": -1.453668236732483,
      "logps/rejected": -1.8302783966064453,
      "loss": 0.482,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.453668236732483,
      "rewards/margins": 0.37661033868789673,
      "rewards/rejected": -1.8302783966064453,
      "sft_loss": 1.5746830701828003,
      "step": 2605
    },
    {
      "epoch": 1.3968891118916207,
      "grad_norm": 10.862884100539372,
      "learning_rate": 6.45058504694559e-07,
      "logits/chosen": 0.047561369836330414,
      "logits/rejected": 0.13421495258808136,
      "logps/chosen": -1.5060184001922607,
      "logps/rejected": -1.8954282999038696,
      "loss": 0.5254,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.5060184001922607,
      "rewards/margins": 0.3894098699092865,
      "rewards/rejected": -1.8954282999038696,
      "sft_loss": 1.605099081993103,
      "step": 2610
    },
    {
      "epoch": 1.3995651446730222,
      "grad_norm": 13.4304998996284,
      "learning_rate": 6.435673828105564e-07,
      "logits/chosen": -0.08659394830465317,
      "logits/rejected": 0.07501087337732315,
      "logps/chosen": -1.3954126834869385,
      "logps/rejected": -1.8897117376327515,
      "loss": 0.4689,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.3954126834869385,
      "rewards/margins": 0.49429893493652344,
      "rewards/rejected": -1.8897117376327515,
      "sft_loss": 1.5091125965118408,
      "step": 2615
    },
    {
      "epoch": 1.402241177454424,
      "grad_norm": 6.402499565877001,
      "learning_rate": 6.420748680351763e-07,
      "logits/chosen": -0.0855594277381897,
      "logits/rejected": -0.08354773372411728,
      "logps/chosen": -1.53382408618927,
      "logps/rejected": -1.7779781818389893,
      "loss": 0.5514,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.53382408618927,
      "rewards/margins": 0.2441541850566864,
      "rewards/rejected": -1.7779781818389893,
      "sft_loss": 1.613317847251892,
      "step": 2620
    },
    {
      "epoch": 1.4049172102358254,
      "grad_norm": 11.188739880849315,
      "learning_rate": 6.405809748488032e-07,
      "logits/chosen": -0.018711065873503685,
      "logits/rejected": 0.15435022115707397,
      "logps/chosen": -1.4651854038238525,
      "logps/rejected": -1.893734335899353,
      "loss": 0.504,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4651854038238525,
      "rewards/margins": 0.42854875326156616,
      "rewards/rejected": -1.893734335899353,
      "sft_loss": 1.4950653314590454,
      "step": 2625
    },
    {
      "epoch": 1.4075932430172269,
      "grad_norm": 8.058799091808403,
      "learning_rate": 6.390857177451956e-07,
      "logits/chosen": -0.1831529587507248,
      "logits/rejected": 0.02995426021516323,
      "logps/chosen": -1.525868535041809,
      "logps/rejected": -1.9160089492797852,
      "loss": 0.5117,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.525868535041809,
      "rewards/margins": 0.3901405334472656,
      "rewards/rejected": -1.9160089492797852,
      "sft_loss": 1.5806645154953003,
      "step": 2630
    },
    {
      "epoch": 1.4102692757986286,
      "grad_norm": 10.003547271148747,
      "learning_rate": 6.375891112313445e-07,
      "logits/chosen": -0.0974530428647995,
      "logits/rejected": -0.0012318312656134367,
      "logps/chosen": -1.4436627626419067,
      "logps/rejected": -1.8610448837280273,
      "loss": 0.4831,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4436627626419067,
      "rewards/margins": 0.41738200187683105,
      "rewards/rejected": -1.8610448837280273,
      "sft_loss": 1.5641653537750244,
      "step": 2635
    },
    {
      "epoch": 1.41294530858003,
      "grad_norm": 4.95988489497464,
      "learning_rate": 6.360911698273326e-07,
      "logits/chosen": -0.04262874275445938,
      "logits/rejected": 0.05346344783902168,
      "logps/chosen": -1.5752532482147217,
      "logps/rejected": -1.998945951461792,
      "loss": 0.507,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5752532482147217,
      "rewards/margins": 0.4236927628517151,
      "rewards/rejected": -1.998945951461792,
      "sft_loss": 1.6537303924560547,
      "step": 2640
    },
    {
      "epoch": 1.4156213413614318,
      "grad_norm": 9.02503655381388,
      "learning_rate": 6.345919080661944e-07,
      "logits/chosen": -0.08658730983734131,
      "logits/rejected": 0.009633921086788177,
      "logps/chosen": -1.4610365629196167,
      "logps/rejected": -1.9266433715820312,
      "loss": 0.4973,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4610365629196167,
      "rewards/margins": 0.4656066298484802,
      "rewards/rejected": -1.9266433715820312,
      "sft_loss": 1.507159948348999,
      "step": 2645
    },
    {
      "epoch": 1.4182973741428333,
      "grad_norm": 5.88129327721945,
      "learning_rate": 6.330913404937737e-07,
      "logits/chosen": -0.17061766982078552,
      "logits/rejected": -0.0011933178175240755,
      "logps/chosen": -1.490103006362915,
      "logps/rejected": -2.159802198410034,
      "loss": 0.4592,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.490103006362915,
      "rewards/margins": 0.6696991920471191,
      "rewards/rejected": -2.159802198410034,
      "sft_loss": 1.5680917501449585,
      "step": 2650
    },
    {
      "epoch": 1.4209734069242348,
      "grad_norm": 7.396741228216022,
      "learning_rate": 6.315894816685838e-07,
      "logits/chosen": -0.06351391226053238,
      "logits/rejected": 0.1041162982583046,
      "logps/chosen": -1.3809651136398315,
      "logps/rejected": -1.8353707790374756,
      "loss": 0.4499,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3809651136398315,
      "rewards/margins": 0.45440563559532166,
      "rewards/rejected": -1.8353707790374756,
      "sft_loss": 1.4856082201004028,
      "step": 2655
    },
    {
      "epoch": 1.4236494397056365,
      "grad_norm": 8.178732535368553,
      "learning_rate": 6.300863461616657e-07,
      "logits/chosen": -0.004477344453334808,
      "logits/rejected": 0.054392505437135696,
      "logps/chosen": -1.3770705461502075,
      "logps/rejected": -1.8496062755584717,
      "loss": 0.4751,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3770705461502075,
      "rewards/margins": 0.47253578901290894,
      "rewards/rejected": -1.8496062755584717,
      "sft_loss": 1.4481415748596191,
      "step": 2660
    },
    {
      "epoch": 1.426325472487038,
      "grad_norm": 6.186302174172172,
      "learning_rate": 6.285819485564465e-07,
      "logits/chosen": -0.18965403735637665,
      "logits/rejected": -0.05282963067293167,
      "logps/chosen": -1.4871896505355835,
      "logps/rejected": -1.9702796936035156,
      "loss": 0.4646,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4871896505355835,
      "rewards/margins": 0.4830901622772217,
      "rewards/rejected": -1.9702796936035156,
      "sft_loss": 1.6419436931610107,
      "step": 2665
    },
    {
      "epoch": 1.4290015052684395,
      "grad_norm": 10.451953780852376,
      "learning_rate": 6.270763034485986e-07,
      "logits/chosen": -0.04714593291282654,
      "logits/rejected": 0.06453403830528259,
      "logps/chosen": -1.6118723154067993,
      "logps/rejected": -1.9500125646591187,
      "loss": 0.531,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.6118723154067993,
      "rewards/margins": 0.3381403386592865,
      "rewards/rejected": -1.9500125646591187,
      "sft_loss": 1.6536245346069336,
      "step": 2670
    },
    {
      "epoch": 1.4316775380498412,
      "grad_norm": 8.909876566015209,
      "learning_rate": 6.255694254458972e-07,
      "logits/chosen": -0.10542843490839005,
      "logits/rejected": 0.0743652805685997,
      "logps/chosen": -1.531923770904541,
      "logps/rejected": -1.9045947790145874,
      "loss": 0.548,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.531923770904541,
      "rewards/margins": 0.3726711869239807,
      "rewards/rejected": -1.9045947790145874,
      "sft_loss": 1.4829033613204956,
      "step": 2675
    },
    {
      "epoch": 1.4343535708312427,
      "grad_norm": 6.795360694229123,
      "learning_rate": 6.240613291680795e-07,
      "logits/chosen": -0.0952523946762085,
      "logits/rejected": 0.09471447020769119,
      "logps/chosen": -1.4994065761566162,
      "logps/rejected": -1.892115831375122,
      "loss": 0.5021,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4994065761566162,
      "rewards/margins": 0.3927091658115387,
      "rewards/rejected": -1.892115831375122,
      "sft_loss": 1.588990569114685,
      "step": 2680
    },
    {
      "epoch": 1.4370296036126442,
      "grad_norm": 7.164348437941818,
      "learning_rate": 6.225520292467021e-07,
      "logits/chosen": -0.10167787969112396,
      "logits/rejected": 0.13535355031490326,
      "logps/chosen": -1.445357084274292,
      "logps/rejected": -1.8391374349594116,
      "loss": 0.4784,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.445357084274292,
      "rewards/margins": 0.3937804102897644,
      "rewards/rejected": -1.8391374349594116,
      "sft_loss": 1.4996330738067627,
      "step": 2685
    },
    {
      "epoch": 1.439705636394046,
      "grad_norm": 17.290840299053265,
      "learning_rate": 6.210415403249993e-07,
      "logits/chosen": -0.22392210364341736,
      "logits/rejected": 0.04952241852879524,
      "logps/chosen": -1.4616634845733643,
      "logps/rejected": -2.038735866546631,
      "loss": 0.4518,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4616634845733643,
      "rewards/margins": 0.5770725011825562,
      "rewards/rejected": -2.038735866546631,
      "sft_loss": 1.5287370681762695,
      "step": 2690
    },
    {
      "epoch": 1.4423816691754474,
      "grad_norm": 5.606463831178428,
      "learning_rate": 6.195298770577415e-07,
      "logits/chosen": -0.021728316321969032,
      "logits/rejected": 0.015017149038612843,
      "logps/chosen": -1.4510247707366943,
      "logps/rejected": -1.9158073663711548,
      "loss": 0.493,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4510247707366943,
      "rewards/margins": 0.4647824764251709,
      "rewards/rejected": -1.9158073663711548,
      "sft_loss": 1.5007692575454712,
      "step": 2695
    },
    {
      "epoch": 1.445057701956849,
      "grad_norm": 7.192489826017525,
      "learning_rate": 6.180170541110923e-07,
      "logits/chosen": -0.1267596036195755,
      "logits/rejected": 0.0693402960896492,
      "logps/chosen": -1.5120747089385986,
      "logps/rejected": -1.962651014328003,
      "loss": 0.4876,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5120747089385986,
      "rewards/margins": 0.45057615637779236,
      "rewards/rejected": -1.962651014328003,
      "sft_loss": 1.6224653720855713,
      "step": 2700
    },
    {
      "epoch": 1.4477337347382506,
      "grad_norm": 5.613140069940458,
      "learning_rate": 6.165030861624663e-07,
      "logits/chosen": -0.16893896460533142,
      "logits/rejected": 0.06323234736919403,
      "logps/chosen": -1.3807225227355957,
      "logps/rejected": -1.9726345539093018,
      "loss": 0.4286,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3807225227355957,
      "rewards/margins": 0.5919119715690613,
      "rewards/rejected": -1.9726345539093018,
      "sft_loss": 1.4009912014007568,
      "step": 2705
    },
    {
      "epoch": 1.4504097675196521,
      "grad_norm": 8.75005969074735,
      "learning_rate": 6.149879879003876e-07,
      "logits/chosen": -0.02317567728459835,
      "logits/rejected": 0.009615430608391762,
      "logps/chosen": -1.4695868492126465,
      "logps/rejected": -1.9241955280303955,
      "loss": 0.4918,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4695868492126465,
      "rewards/margins": 0.45460858941078186,
      "rewards/rejected": -1.9241955280303955,
      "sft_loss": 1.501080870628357,
      "step": 2710
    },
    {
      "epoch": 1.4530858003010536,
      "grad_norm": 4.506798373707267,
      "learning_rate": 6.13471774024346e-07,
      "logits/chosen": -0.19773828983306885,
      "logits/rejected": -0.07556317001581192,
      "logps/chosen": -1.377198576927185,
      "logps/rejected": -1.7756855487823486,
      "loss": 0.4903,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.377198576927185,
      "rewards/margins": 0.3984869718551636,
      "rewards/rejected": -1.7756855487823486,
      "sft_loss": 1.522790789604187,
      "step": 2715
    },
    {
      "epoch": 1.4557618330824553,
      "grad_norm": 7.052556075358587,
      "learning_rate": 6.119544592446551e-07,
      "logits/chosen": -0.1464635580778122,
      "logits/rejected": -0.015275077894330025,
      "logps/chosen": -1.4300029277801514,
      "logps/rejected": -1.7926452159881592,
      "loss": 0.4887,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4300029277801514,
      "rewards/margins": 0.3626423180103302,
      "rewards/rejected": -1.7926452159881592,
      "sft_loss": 1.482043981552124,
      "step": 2720
    },
    {
      "epoch": 1.4584378658638568,
      "grad_norm": 6.510991794442674,
      "learning_rate": 6.104360582823096e-07,
      "logits/chosen": -0.10228048264980316,
      "logits/rejected": 0.024892430752515793,
      "logps/chosen": -1.4259611368179321,
      "logps/rejected": -1.8754980564117432,
      "loss": 0.4684,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4259611368179321,
      "rewards/margins": 0.44953688979148865,
      "rewards/rejected": -1.8754980564117432,
      "sft_loss": 1.4861032962799072,
      "step": 2725
    },
    {
      "epoch": 1.4611138986452583,
      "grad_norm": 7.365459217470868,
      "learning_rate": 6.089165858688423e-07,
      "logits/chosen": -0.14073030650615692,
      "logits/rejected": 0.06208374351263046,
      "logps/chosen": -1.4707056283950806,
      "logps/rejected": -1.9706439971923828,
      "loss": 0.4797,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4707056283950806,
      "rewards/margins": 0.4999384880065918,
      "rewards/rejected": -1.9706439971923828,
      "sft_loss": 1.5462729930877686,
      "step": 2730
    },
    {
      "epoch": 1.46378993142666,
      "grad_norm": 6.119979287546052,
      "learning_rate": 6.073960567461811e-07,
      "logits/chosen": -0.1096295490860939,
      "logits/rejected": 0.11302530765533447,
      "logps/chosen": -1.3502978086471558,
      "logps/rejected": -1.9090697765350342,
      "loss": 0.4295,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3502978086471558,
      "rewards/margins": 0.5587717890739441,
      "rewards/rejected": -1.9090697765350342,
      "sft_loss": 1.4756377935409546,
      "step": 2735
    },
    {
      "epoch": 1.4664659642080615,
      "grad_norm": 8.220082955508166,
      "learning_rate": 6.058744856665065e-07,
      "logits/chosen": -0.1352849304676056,
      "logits/rejected": 0.0010965674882754683,
      "logps/chosen": -1.4500648975372314,
      "logps/rejected": -2.068101406097412,
      "loss": 0.4469,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4500648975372314,
      "rewards/margins": 0.6180363893508911,
      "rewards/rejected": -2.068101406097412,
      "sft_loss": 1.5455083847045898,
      "step": 2740
    },
    {
      "epoch": 1.469141996989463,
      "grad_norm": 6.1082001469248,
      "learning_rate": 6.043518873921074e-07,
      "logits/chosen": -0.15699729323387146,
      "logits/rejected": 0.010936413891613483,
      "logps/chosen": -1.3993462324142456,
      "logps/rejected": -1.813114881515503,
      "loss": 0.4811,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.3993462324142456,
      "rewards/margins": 0.4137686789035797,
      "rewards/rejected": -1.813114881515503,
      "sft_loss": 1.472827672958374,
      "step": 2745
    },
    {
      "epoch": 1.4718180297708647,
      "grad_norm": 6.667631595880087,
      "learning_rate": 6.028282766952393e-07,
      "logits/chosen": -0.10414779186248779,
      "logits/rejected": 0.016749560832977295,
      "logps/chosen": -1.547855257987976,
      "logps/rejected": -2.1255764961242676,
      "loss": 0.4501,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.547855257987976,
      "rewards/margins": 0.5777208209037781,
      "rewards/rejected": -2.1255764961242676,
      "sft_loss": 1.6323277950286865,
      "step": 2750
    },
    {
      "epoch": 1.4744940625522662,
      "grad_norm": 8.921386278747955,
      "learning_rate": 6.013036683579798e-07,
      "logits/chosen": -0.04205799847841263,
      "logits/rejected": 0.12203924357891083,
      "logps/chosen": -1.472687005996704,
      "logps/rejected": -1.9264453649520874,
      "loss": 0.4847,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.472687005996704,
      "rewards/margins": 0.4537584185600281,
      "rewards/rejected": -1.9264453649520874,
      "sft_loss": 1.5990089178085327,
      "step": 2755
    },
    {
      "epoch": 1.4771700953336677,
      "grad_norm": 7.053570351514786,
      "learning_rate": 5.997780771720854e-07,
      "logits/chosen": -0.1811997890472412,
      "logits/rejected": 0.017085423693060875,
      "logps/chosen": -1.586169719696045,
      "logps/rejected": -2.2159647941589355,
      "loss": 0.4396,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.586169719696045,
      "rewards/margins": 0.629794716835022,
      "rewards/rejected": -2.2159647941589355,
      "sft_loss": 1.6507060527801514,
      "step": 2760
    },
    {
      "epoch": 1.4798461281150694,
      "grad_norm": 8.264947967152885,
      "learning_rate": 5.982515179388486e-07,
      "logits/chosen": -0.09138262271881104,
      "logits/rejected": 0.0653950497508049,
      "logps/chosen": -1.5182015895843506,
      "logps/rejected": -1.9837443828582764,
      "loss": 0.4573,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5182015895843506,
      "rewards/margins": 0.46554288268089294,
      "rewards/rejected": -1.9837443828582764,
      "sft_loss": 1.6368268728256226,
      "step": 2765
    },
    {
      "epoch": 1.482522160896471,
      "grad_norm": 6.413760604034631,
      "learning_rate": 5.967240054689541e-07,
      "logits/chosen": -0.1785578727722168,
      "logits/rejected": -0.09279811382293701,
      "logps/chosen": -1.5066957473754883,
      "logps/rejected": -1.8460357189178467,
      "loss": 0.5041,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5066957473754883,
      "rewards/margins": 0.33934006094932556,
      "rewards/rejected": -1.8460357189178467,
      "sft_loss": 1.5868940353393555,
      "step": 2770
    },
    {
      "epoch": 1.4851981936778724,
      "grad_norm": 9.701479183254484,
      "learning_rate": 5.951955545823342e-07,
      "logits/chosen": -0.1419348418712616,
      "logits/rejected": -0.08633621037006378,
      "logps/chosen": -1.5398082733154297,
      "logps/rejected": -2.1274075508117676,
      "loss": 0.4736,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5398082733154297,
      "rewards/margins": 0.5875992178916931,
      "rewards/rejected": -2.1274075508117676,
      "sft_loss": 1.6137285232543945,
      "step": 2775
    },
    {
      "epoch": 1.4878742264592741,
      "grad_norm": 5.9921000870979215,
      "learning_rate": 5.936661801080263e-07,
      "logits/chosen": -0.13307741284370422,
      "logits/rejected": -0.019663050770759583,
      "logps/chosen": -1.680048942565918,
      "logps/rejected": -2.104114532470703,
      "loss": 0.5418,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.680048942565918,
      "rewards/margins": 0.4240652918815613,
      "rewards/rejected": -2.104114532470703,
      "sft_loss": 1.6894235610961914,
      "step": 2780
    },
    {
      "epoch": 1.4905502592406756,
      "grad_norm": 13.089147452135542,
      "learning_rate": 5.92135896884028e-07,
      "logits/chosen": -0.19363871216773987,
      "logits/rejected": -0.031053265556693077,
      "logps/chosen": -1.6516166925430298,
      "logps/rejected": -2.150430679321289,
      "loss": 0.5165,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.6516166925430298,
      "rewards/margins": 0.49881380796432495,
      "rewards/rejected": -2.150430679321289,
      "sft_loss": 1.6912845373153687,
      "step": 2785
    },
    {
      "epoch": 1.4932262920220774,
      "grad_norm": 9.129385007733266,
      "learning_rate": 5.906047197571541e-07,
      "logits/chosen": -0.07705952227115631,
      "logits/rejected": -0.09640021622180939,
      "logps/chosen": -1.5163434743881226,
      "logps/rejected": -1.9805564880371094,
      "loss": 0.5026,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5163434743881226,
      "rewards/margins": 0.4642130434513092,
      "rewards/rejected": -1.9805564880371094,
      "sft_loss": 1.6745860576629639,
      "step": 2790
    },
    {
      "epoch": 1.4959023248034788,
      "grad_norm": 6.106860258760642,
      "learning_rate": 5.890726635828919e-07,
      "logits/chosen": 0.02769961953163147,
      "logits/rejected": 0.048429567366838455,
      "logps/chosen": -1.3723499774932861,
      "logps/rejected": -1.8381656408309937,
      "loss": 0.4586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3723499774932861,
      "rewards/margins": 0.46581578254699707,
      "rewards/rejected": -1.8381656408309937,
      "sft_loss": 1.431898593902588,
      "step": 2795
    },
    {
      "epoch": 1.4985783575848803,
      "grad_norm": 10.749893862767616,
      "learning_rate": 5.875397432252569e-07,
      "logits/chosen": -0.184439018368721,
      "logits/rejected": -0.09069575369358063,
      "logps/chosen": -1.565915822982788,
      "logps/rejected": -2.0079472064971924,
      "loss": 0.5266,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.565915822982788,
      "rewards/margins": 0.44203153252601624,
      "rewards/rejected": -2.0079472064971924,
      "sft_loss": 1.6745878458023071,
      "step": 2800
    },
    {
      "epoch": 1.4985783575848803,
      "eval_logits/chosen": 0.195302814245224,
      "eval_logits/rejected": 0.29392915964126587,
      "eval_logps/chosen": -1.5391197204589844,
      "eval_logps/rejected": -2.0192654132843018,
      "eval_loss": 0.49466973543167114,
      "eval_rewards/accuracies": 0.6572700142860413,
      "eval_rewards/chosen": -1.5391197204589844,
      "eval_rewards/margins": 0.4801456332206726,
      "eval_rewards/rejected": -2.0192654132843018,
      "eval_runtime": 43.4371,
      "eval_samples_per_second": 30.964,
      "eval_sft_loss": 1.6154696941375732,
      "eval_steps_per_second": 7.758,
      "step": 2800
    },
    {
      "epoch": 1.5012543903662818,
      "grad_norm": 3.8318594608132135,
      "learning_rate": 5.860059735566491e-07,
      "logits/chosen": -0.2765553593635559,
      "logits/rejected": -0.11357270181179047,
      "logps/chosen": -1.367326259613037,
      "logps/rejected": -1.9252784252166748,
      "loss": 0.426,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.367326259613037,
      "rewards/margins": 0.5579522252082825,
      "rewards/rejected": -1.9252784252166748,
      "sft_loss": 1.488986849784851,
      "step": 2805
    },
    {
      "epoch": 1.5039304231476835,
      "grad_norm": 9.834571664430086,
      "learning_rate": 5.844713694577087e-07,
      "logits/chosen": -0.1395595520734787,
      "logits/rejected": -0.08087310194969177,
      "logps/chosen": -1.48549485206604,
      "logps/rejected": -1.9511163234710693,
      "loss": 0.4772,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.48549485206604,
      "rewards/margins": 0.46562162041664124,
      "rewards/rejected": -1.9511163234710693,
      "sft_loss": 1.6048133373260498,
      "step": 2810
    },
    {
      "epoch": 1.5066064559290853,
      "grad_norm": 6.52049819582276,
      "learning_rate": 5.829359458171714e-07,
      "logits/chosen": -0.07088275253772736,
      "logits/rejected": 0.06389816105365753,
      "logps/chosen": -1.4949347972869873,
      "logps/rejected": -2.0141525268554688,
      "loss": 0.4787,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4949347972869873,
      "rewards/margins": 0.5192179083824158,
      "rewards/rejected": -2.0141525268554688,
      "sft_loss": 1.5503588914871216,
      "step": 2815
    },
    {
      "epoch": 1.5092824887104868,
      "grad_norm": 6.625902983275402,
      "learning_rate": 5.81399717531724e-07,
      "logits/chosen": -0.1074337586760521,
      "logits/rejected": 0.09632369130849838,
      "logps/chosen": -1.5207237005233765,
      "logps/rejected": -1.9671484231948853,
      "loss": 0.4963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5207237005233765,
      "rewards/margins": 0.44642454385757446,
      "rewards/rejected": -1.9671484231948853,
      "sft_loss": 1.576856017112732,
      "step": 2820
    },
    {
      "epoch": 1.5119585214918883,
      "grad_norm": 7.208878180194657,
      "learning_rate": 5.798626995058602e-07,
      "logits/chosen": -0.17939253151416779,
      "logits/rejected": 0.022124072536826134,
      "logps/chosen": -1.5701862573623657,
      "logps/rejected": -2.147313117980957,
      "loss": 0.4768,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5701862573623657,
      "rewards/margins": 0.5771269798278809,
      "rewards/rejected": -2.147313117980957,
      "sft_loss": 1.6021369695663452,
      "step": 2825
    },
    {
      "epoch": 1.51463455427329,
      "grad_norm": 8.871421803027232,
      "learning_rate": 5.783249066517354e-07,
      "logits/chosen": -0.11368022114038467,
      "logits/rejected": 0.05845404416322708,
      "logps/chosen": -1.5595228672027588,
      "logps/rejected": -2.0032458305358887,
      "loss": 0.4964,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5595228672027588,
      "rewards/margins": 0.44372302293777466,
      "rewards/rejected": -2.0032458305358887,
      "sft_loss": 1.6930491924285889,
      "step": 2830
    },
    {
      "epoch": 1.5173105870546915,
      "grad_norm": 6.925181490630755,
      "learning_rate": 5.767863538890228e-07,
      "logits/chosen": -0.11891146749258041,
      "logits/rejected": 0.06260021775960922,
      "logps/chosen": -1.5366853475570679,
      "logps/rejected": -2.0646588802337646,
      "loss": 0.4755,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5366853475570679,
      "rewards/margins": 0.527973473072052,
      "rewards/rejected": -2.0646588802337646,
      "sft_loss": 1.6199018955230713,
      "step": 2835
    },
    {
      "epoch": 1.519986619836093,
      "grad_norm": 7.008280760515188,
      "learning_rate": 5.75247056144768e-07,
      "logits/chosen": -0.1227370947599411,
      "logits/rejected": -0.018377363681793213,
      "logps/chosen": -1.5639212131500244,
      "logps/rejected": -1.9450569152832031,
      "loss": 0.5215,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5639212131500244,
      "rewards/margins": 0.3811357617378235,
      "rewards/rejected": -1.9450569152832031,
      "sft_loss": 1.6612507104873657,
      "step": 2840
    },
    {
      "epoch": 1.5226626526174947,
      "grad_norm": 9.088827613891114,
      "learning_rate": 5.737070283532444e-07,
      "logits/chosen": -0.08626069128513336,
      "logits/rejected": 0.013733291998505592,
      "logps/chosen": -1.5527054071426392,
      "logps/rejected": -2.0260472297668457,
      "loss": 0.5172,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.5527054071426392,
      "rewards/margins": 0.47334200143814087,
      "rewards/rejected": -2.0260472297668457,
      "sft_loss": 1.552793264389038,
      "step": 2845
    },
    {
      "epoch": 1.5253386853988962,
      "grad_norm": 6.831388808036328,
      "learning_rate": 5.721662854558084e-07,
      "logits/chosen": -0.1691809594631195,
      "logits/rejected": -0.0750095397233963,
      "logps/chosen": -1.5391981601715088,
      "logps/rejected": -2.047201633453369,
      "loss": 0.4935,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5391981601715088,
      "rewards/margins": 0.5080040693283081,
      "rewards/rejected": -2.047201633453369,
      "sft_loss": 1.60979425907135,
      "step": 2850
    },
    {
      "epoch": 1.5280147181802977,
      "grad_norm": 7.175477010957574,
      "learning_rate": 5.706248424007545e-07,
      "logits/chosen": -0.14673514664173126,
      "logits/rejected": 0.0352972038090229,
      "logps/chosen": -1.705213189125061,
      "logps/rejected": -2.234771966934204,
      "loss": 0.4749,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.705213189125061,
      "rewards/margins": 0.5295583605766296,
      "rewards/rejected": -2.234771966934204,
      "sft_loss": 1.800531029701233,
      "step": 2855
    },
    {
      "epoch": 1.5306907509616994,
      "grad_norm": 7.499618017199612,
      "learning_rate": 5.690827141431699e-07,
      "logits/chosen": -0.21774613857269287,
      "logits/rejected": -0.020393937826156616,
      "logps/chosen": -1.4769175052642822,
      "logps/rejected": -1.9039077758789062,
      "loss": 0.4746,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4769175052642822,
      "rewards/margins": 0.4269903600215912,
      "rewards/rejected": -1.9039077758789062,
      "sft_loss": 1.5506912469863892,
      "step": 2860
    },
    {
      "epoch": 1.5333667837431009,
      "grad_norm": 6.416943109466104,
      "learning_rate": 5.675399156447897e-07,
      "logits/chosen": -0.25113674998283386,
      "logits/rejected": -0.10772594064474106,
      "logps/chosen": -1.522179365158081,
      "logps/rejected": -1.9736213684082031,
      "loss": 0.4688,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.522179365158081,
      "rewards/margins": 0.4514419436454773,
      "rewards/rejected": -1.9736213684082031,
      "sft_loss": 1.649560570716858,
      "step": 2865
    },
    {
      "epoch": 1.5360428165245024,
      "grad_norm": 9.949473471115484,
      "learning_rate": 5.659964618738515e-07,
      "logits/chosen": -0.13834629952907562,
      "logits/rejected": -0.013131308369338512,
      "logps/chosen": -1.5474927425384521,
      "logps/rejected": -1.873211145401001,
      "loss": 0.515,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.5474927425384521,
      "rewards/margins": 0.32571840286254883,
      "rewards/rejected": -1.873211145401001,
      "sft_loss": 1.581970453262329,
      "step": 2870
    },
    {
      "epoch": 1.538718849305904,
      "grad_norm": 7.615723711948691,
      "learning_rate": 5.644523678049509e-07,
      "logits/chosen": -0.15234772861003876,
      "logits/rejected": -0.038485340774059296,
      "logps/chosen": -1.5660016536712646,
      "logps/rejected": -2.0160014629364014,
      "loss": 0.4617,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5660016536712646,
      "rewards/margins": 0.4499998092651367,
      "rewards/rejected": -2.0160014629364014,
      "sft_loss": 1.5764434337615967,
      "step": 2875
    },
    {
      "epoch": 1.5413948820873056,
      "grad_norm": 8.829175592430431,
      "learning_rate": 5.629076484188952e-07,
      "logits/chosen": 0.002240369562059641,
      "logits/rejected": 0.12692388892173767,
      "logps/chosen": -1.4671860933303833,
      "logps/rejected": -1.986998200416565,
      "loss": 0.4817,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4671860933303833,
      "rewards/margins": 0.5198121666908264,
      "rewards/rejected": -1.986998200416565,
      "sft_loss": 1.5353573560714722,
      "step": 2880
    },
    {
      "epoch": 1.544070914868707,
      "grad_norm": 8.050631038655702,
      "learning_rate": 5.613623187025587e-07,
      "logits/chosen": -0.13125446438789368,
      "logits/rejected": -0.0016770787769928575,
      "logps/chosen": -1.6214059591293335,
      "logps/rejected": -2.1163296699523926,
      "loss": 0.4903,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6214059591293335,
      "rewards/margins": 0.4949234127998352,
      "rewards/rejected": -2.1163296699523926,
      "sft_loss": 1.7011600732803345,
      "step": 2885
    },
    {
      "epoch": 1.5467469476501088,
      "grad_norm": 7.662816680327483,
      "learning_rate": 5.598163936487369e-07,
      "logits/chosen": -0.202124685049057,
      "logits/rejected": 0.001256814575754106,
      "logps/chosen": -1.5663325786590576,
      "logps/rejected": -2.1145505905151367,
      "loss": 0.4669,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5663325786590576,
      "rewards/margins": 0.5482179522514343,
      "rewards/rejected": -2.1145505905151367,
      "sft_loss": 1.6170275211334229,
      "step": 2890
    },
    {
      "epoch": 1.5494229804315103,
      "grad_norm": 8.485306367150564,
      "learning_rate": 5.582698882560017e-07,
      "logits/chosen": -0.17051918804645538,
      "logits/rejected": -0.007482844404876232,
      "logps/chosen": -1.4477908611297607,
      "logps/rejected": -1.9668010473251343,
      "loss": 0.4643,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4477908611297607,
      "rewards/margins": 0.5190101861953735,
      "rewards/rejected": -1.9668010473251343,
      "sft_loss": 1.4976394176483154,
      "step": 2895
    },
    {
      "epoch": 1.5520990132129118,
      "grad_norm": 5.4358669169930005,
      "learning_rate": 5.567228175285549e-07,
      "logits/chosen": -0.0875912755727768,
      "logits/rejected": 0.02966555580496788,
      "logps/chosen": -1.5256271362304688,
      "logps/rejected": -2.0319604873657227,
      "loss": 0.4669,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5256271362304688,
      "rewards/margins": 0.5063332319259644,
      "rewards/rejected": -2.0319604873657227,
      "sft_loss": 1.6018095016479492,
      "step": 2900
    },
    {
      "epoch": 1.5547750459943135,
      "grad_norm": 6.746406325493352,
      "learning_rate": 5.551751964760838e-07,
      "logits/chosen": -0.011767564341425896,
      "logits/rejected": 0.01628190465271473,
      "logps/chosen": -1.49928879737854,
      "logps/rejected": -1.9732303619384766,
      "loss": 0.4956,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.49928879737854,
      "rewards/margins": 0.4739416241645813,
      "rewards/rejected": -1.9732303619384766,
      "sft_loss": 1.584179162979126,
      "step": 2905
    },
    {
      "epoch": 1.557451078775715,
      "grad_norm": 6.6605557708847005,
      "learning_rate": 5.536270401136145e-07,
      "logits/chosen": -0.08847250044345856,
      "logits/rejected": 0.022497396916151047,
      "logps/chosen": -1.51120126247406,
      "logps/rejected": -1.901601791381836,
      "loss": 0.4866,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.51120126247406,
      "rewards/margins": 0.3904004693031311,
      "rewards/rejected": -1.901601791381836,
      "sft_loss": 1.6852781772613525,
      "step": 2910
    },
    {
      "epoch": 1.5601271115571165,
      "grad_norm": 15.629320305461311,
      "learning_rate": 5.520783634613667e-07,
      "logits/chosen": -0.03065609000623226,
      "logits/rejected": 0.16193042695522308,
      "logps/chosen": -1.5821765661239624,
      "logps/rejected": -2.097710132598877,
      "loss": 0.5082,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5821765661239624,
      "rewards/margins": 0.515533447265625,
      "rewards/rejected": -2.097710132598877,
      "sft_loss": 1.6894975900650024,
      "step": 2915
    },
    {
      "epoch": 1.5628031443385182,
      "grad_norm": 8.017551981264566,
      "learning_rate": 5.505291815446082e-07,
      "logits/chosen": -0.03545909374952316,
      "logits/rejected": 0.10086387395858765,
      "logps/chosen": -1.5606505870819092,
      "logps/rejected": -2.073725461959839,
      "loss": 0.4878,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5606505870819092,
      "rewards/margins": 0.5130751132965088,
      "rewards/rejected": -2.073725461959839,
      "sft_loss": 1.6506057977676392,
      "step": 2920
    },
    {
      "epoch": 1.5654791771199197,
      "grad_norm": 7.474124977180021,
      "learning_rate": 5.489795093935089e-07,
      "logits/chosen": -0.019739773124456406,
      "logits/rejected": 0.05907121300697327,
      "logps/chosen": -1.432644009590149,
      "logps/rejected": -1.9527826309204102,
      "loss": 0.4903,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.432644009590149,
      "rewards/margins": 0.520138680934906,
      "rewards/rejected": -1.9527826309204102,
      "sft_loss": 1.4820513725280762,
      "step": 2925
    },
    {
      "epoch": 1.5681552099013212,
      "grad_norm": 7.125622116791099,
      "learning_rate": 5.474293620429946e-07,
      "logits/chosen": -0.17671312391757965,
      "logits/rejected": 0.02326101064682007,
      "logps/chosen": -1.4796098470687866,
      "logps/rejected": -2.236234664916992,
      "loss": 0.431,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4796098470687866,
      "rewards/margins": 0.7566248774528503,
      "rewards/rejected": -2.236234664916992,
      "sft_loss": 1.6504055261611938,
      "step": 2930
    },
    {
      "epoch": 1.570831242682723,
      "grad_norm": 8.608945768862661,
      "learning_rate": 5.458787545326018e-07,
      "logits/chosen": -0.14483094215393066,
      "logits/rejected": 0.013085213489830494,
      "logps/chosen": -1.562831163406372,
      "logps/rejected": -2.0342061519622803,
      "loss": 0.4888,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.562831163406372,
      "rewards/margins": 0.47137507796287537,
      "rewards/rejected": -2.0342061519622803,
      "sft_loss": 1.6741364002227783,
      "step": 2935
    },
    {
      "epoch": 1.5735072754641244,
      "grad_norm": 7.206925970451456,
      "learning_rate": 5.443277019063311e-07,
      "logits/chosen": -0.1183919757604599,
      "logits/rejected": 0.0802488699555397,
      "logps/chosen": -1.523560881614685,
      "logps/rejected": -2.1156466007232666,
      "loss": 0.4657,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.523560881614685,
      "rewards/margins": 0.5920857191085815,
      "rewards/rejected": -2.1156466007232666,
      "sft_loss": 1.641283631324768,
      "step": 2940
    },
    {
      "epoch": 1.5761833082455259,
      "grad_norm": 10.318213241407362,
      "learning_rate": 5.427762192125023e-07,
      "logits/chosen": -0.1034453734755516,
      "logits/rejected": 0.05875151604413986,
      "logps/chosen": -1.4753280878067017,
      "logps/rejected": -1.8768761157989502,
      "loss": 0.4977,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4753280878067017,
      "rewards/margins": 0.40154796838760376,
      "rewards/rejected": -1.8768761157989502,
      "sft_loss": 1.526484727859497,
      "step": 2945
    },
    {
      "epoch": 1.5788593410269276,
      "grad_norm": 10.426324372194873,
      "learning_rate": 5.41224321503607e-07,
      "logits/chosen": -0.008512385189533234,
      "logits/rejected": 0.2818513810634613,
      "logps/chosen": -1.4411323070526123,
      "logps/rejected": -2.0921225547790527,
      "loss": 0.4294,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4411323070526123,
      "rewards/margins": 0.6509900689125061,
      "rewards/rejected": -2.0921225547790527,
      "sft_loss": 1.5098687410354614,
      "step": 2950
    },
    {
      "epoch": 1.5815353738083293,
      "grad_norm": 8.73690464393858,
      "learning_rate": 5.396720238361637e-07,
      "logits/chosen": -0.020450295880436897,
      "logits/rejected": 0.09912039339542389,
      "logps/chosen": -1.5450700521469116,
      "logps/rejected": -2.082106113433838,
      "loss": 0.4816,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5450700521469116,
      "rewards/margins": 0.5370359420776367,
      "rewards/rejected": -2.082106113433838,
      "sft_loss": 1.6611696481704712,
      "step": 2955
    },
    {
      "epoch": 1.5842114065897306,
      "grad_norm": 8.984068771825296,
      "learning_rate": 5.381193412705711e-07,
      "logits/chosen": -0.12652283906936646,
      "logits/rejected": 0.02124221995472908,
      "logps/chosen": -1.5176342725753784,
      "logps/rejected": -1.9752323627471924,
      "loss": 0.4832,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5176342725753784,
      "rewards/margins": 0.4575978219509125,
      "rewards/rejected": -1.9752323627471924,
      "sft_loss": 1.5860871076583862,
      "step": 2960
    },
    {
      "epoch": 1.5868874393711323,
      "grad_norm": 6.047546625096609,
      "learning_rate": 5.365662888709622e-07,
      "logits/chosen": -0.07259048521518707,
      "logits/rejected": 0.034884002059698105,
      "logps/chosen": -1.4993550777435303,
      "logps/rejected": -1.9948831796646118,
      "loss": 0.4932,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4993550777435303,
      "rewards/margins": 0.4955279231071472,
      "rewards/rejected": -1.9948831796646118,
      "sft_loss": 1.558639407157898,
      "step": 2965
    },
    {
      "epoch": 1.589563472152534,
      "grad_norm": 10.15818250535398,
      "learning_rate": 5.350128817050585e-07,
      "logits/chosen": -0.10375696420669556,
      "logits/rejected": 0.12088628858327866,
      "logps/chosen": -1.610216498374939,
      "logps/rejected": -2.1483049392700195,
      "loss": 0.5139,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.610216498374939,
      "rewards/margins": 0.5380884408950806,
      "rewards/rejected": -2.1483049392700195,
      "sft_loss": 1.7288093566894531,
      "step": 2970
    },
    {
      "epoch": 1.5922395049339353,
      "grad_norm": 8.730149993908,
      "learning_rate": 5.334591348440229e-07,
      "logits/chosen": -0.07916657626628876,
      "logits/rejected": 0.09168480336666107,
      "logps/chosen": -1.5467594861984253,
      "logps/rejected": -2.1468772888183594,
      "loss": 0.4628,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5467594861984253,
      "rewards/margins": 0.6001176238059998,
      "rewards/rejected": -2.1468772888183594,
      "sft_loss": 1.657918930053711,
      "step": 2975
    },
    {
      "epoch": 1.594915537715337,
      "grad_norm": 5.4880649780877,
      "learning_rate": 5.319050633623141e-07,
      "logits/chosen": -0.15298187732696533,
      "logits/rejected": 0.04283507913351059,
      "logps/chosen": -1.6657731533050537,
      "logps/rejected": -2.1596436500549316,
      "loss": 0.4795,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.6657731533050537,
      "rewards/margins": 0.49387043714523315,
      "rewards/rejected": -2.1596436500549316,
      "sft_loss": 1.7058131694793701,
      "step": 2980
    },
    {
      "epoch": 1.5975915704967387,
      "grad_norm": 6.2781301998920735,
      "learning_rate": 5.303506823375409e-07,
      "logits/chosen": -0.12671543657779694,
      "logits/rejected": 0.09902218729257584,
      "logps/chosen": -1.6928863525390625,
      "logps/rejected": -2.257063388824463,
      "loss": 0.5153,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6928863525390625,
      "rewards/margins": 0.5641769766807556,
      "rewards/rejected": -2.257063388824463,
      "sft_loss": 1.6770423650741577,
      "step": 2985
    },
    {
      "epoch": 1.60026760327814,
      "grad_norm": 10.670619321805976,
      "learning_rate": 5.287960068503143e-07,
      "logits/chosen": -0.10209260880947113,
      "logits/rejected": 0.11269841343164444,
      "logps/chosen": -1.517526388168335,
      "logps/rejected": -2.184566020965576,
      "loss": 0.4519,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.517526388168335,
      "rewards/margins": 0.667039692401886,
      "rewards/rejected": -2.184566020965576,
      "sft_loss": 1.5879290103912354,
      "step": 2990
    },
    {
      "epoch": 1.6029436360595417,
      "grad_norm": 20.760880204872265,
      "learning_rate": 5.272410519841032e-07,
      "logits/chosen": -0.08403249830007553,
      "logits/rejected": 0.056710999459028244,
      "logps/chosen": -1.6829345226287842,
      "logps/rejected": -2.4216763973236084,
      "loss": 0.467,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.6829345226287842,
      "rewards/margins": 0.7387418150901794,
      "rewards/rejected": -2.4216763973236084,
      "sft_loss": 1.8225390911102295,
      "step": 2995
    },
    {
      "epoch": 1.6056196688409434,
      "grad_norm": 4.916358646100815,
      "learning_rate": 5.256858328250861e-07,
      "logits/chosen": -0.10212262719869614,
      "logits/rejected": 0.07346180081367493,
      "logps/chosen": -1.5672862529754639,
      "logps/rejected": -2.046978712081909,
      "loss": 0.4964,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5672862529754639,
      "rewards/margins": 0.4796925485134125,
      "rewards/rejected": -2.046978712081909,
      "sft_loss": 1.574955701828003,
      "step": 3000
    },
    {
      "epoch": 1.608295701622345,
      "grad_norm": 11.006061549135756,
      "learning_rate": 5.241303644620063e-07,
      "logits/chosen": -0.18173246085643768,
      "logits/rejected": -0.011754069477319717,
      "logps/chosen": -1.4685728549957275,
      "logps/rejected": -1.8448057174682617,
      "loss": 0.4863,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4685728549957275,
      "rewards/margins": 0.3762328326702118,
      "rewards/rejected": -1.8448057174682617,
      "sft_loss": 1.5300872325897217,
      "step": 3005
    },
    {
      "epoch": 1.6109717344037464,
      "grad_norm": 7.797550997747095,
      "learning_rate": 5.225746619860248e-07,
      "logits/chosen": -0.158115416765213,
      "logits/rejected": -0.010147708468139172,
      "logps/chosen": -1.4935801029205322,
      "logps/rejected": -1.8723909854888916,
      "loss": 0.5091,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4935801029205322,
      "rewards/margins": 0.37881091237068176,
      "rewards/rejected": -1.8723909854888916,
      "sft_loss": 1.5578104257583618,
      "step": 3010
    },
    {
      "epoch": 1.6136477671851481,
      "grad_norm": 10.014579651548331,
      "learning_rate": 5.210187404905735e-07,
      "logits/chosen": 0.06347858905792236,
      "logits/rejected": 0.15640303492546082,
      "logps/chosen": -1.4985144138336182,
      "logps/rejected": -1.868246078491211,
      "loss": 0.4968,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4985144138336182,
      "rewards/margins": 0.36973172426223755,
      "rewards/rejected": -1.868246078491211,
      "sft_loss": 1.5547329187393188,
      "step": 3015
    },
    {
      "epoch": 1.6163237999665496,
      "grad_norm": 6.684457081770787,
      "learning_rate": 5.194626150712098e-07,
      "logits/chosen": -0.13590192794799805,
      "logits/rejected": 0.03512415289878845,
      "logps/chosen": -1.483924388885498,
      "logps/rejected": -1.9175758361816406,
      "loss": 0.4668,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.483924388885498,
      "rewards/margins": 0.4336513578891754,
      "rewards/rejected": -1.9175758361816406,
      "sft_loss": 1.6244289875030518,
      "step": 3020
    },
    {
      "epoch": 1.6189998327479511,
      "grad_norm": 6.5624449173763955,
      "learning_rate": 5.179063008254695e-07,
      "logits/chosen": -0.08367796242237091,
      "logits/rejected": 0.10011348873376846,
      "logps/chosen": -1.4311835765838623,
      "logps/rejected": -1.8682610988616943,
      "loss": 0.4688,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4311835765838623,
      "rewards/margins": 0.4370775818824768,
      "rewards/rejected": -1.8682610988616943,
      "sft_loss": 1.5716338157653809,
      "step": 3025
    },
    {
      "epoch": 1.6216758655293528,
      "grad_norm": 7.1403128736198545,
      "learning_rate": 5.163498128527199e-07,
      "logits/chosen": -0.06696267426013947,
      "logits/rejected": 0.09421399235725403,
      "logps/chosen": -1.5920861959457397,
      "logps/rejected": -2.0236916542053223,
      "loss": 0.5067,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5920861959457397,
      "rewards/margins": 0.43160510063171387,
      "rewards/rejected": -2.0236916542053223,
      "sft_loss": 1.6656211614608765,
      "step": 3030
    },
    {
      "epoch": 1.6243518983107543,
      "grad_norm": 8.899740715118822,
      "learning_rate": 5.147931662540144e-07,
      "logits/chosen": 0.04497329145669937,
      "logits/rejected": 0.1868835836648941,
      "logps/chosen": -1.5424516201019287,
      "logps/rejected": -1.8982648849487305,
      "loss": 0.4992,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5424516201019287,
      "rewards/margins": 0.355813205242157,
      "rewards/rejected": -1.8982648849487305,
      "sft_loss": 1.6119035482406616,
      "step": 3035
    },
    {
      "epoch": 1.6270279310921558,
      "grad_norm": 6.559493814754457,
      "learning_rate": 5.132363761319449e-07,
      "logits/chosen": -0.0907437801361084,
      "logits/rejected": -0.013587561435997486,
      "logps/chosen": -1.4611659049987793,
      "logps/rejected": -2.103156328201294,
      "loss": 0.4392,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4611659049987793,
      "rewards/margins": 0.6419903635978699,
      "rewards/rejected": -2.103156328201294,
      "sft_loss": 1.5914957523345947,
      "step": 3040
    },
    {
      "epoch": 1.6297039638735575,
      "grad_norm": 14.776232738852986,
      "learning_rate": 5.116794575904962e-07,
      "logits/chosen": -0.06494507193565369,
      "logits/rejected": 0.05210378021001816,
      "logps/chosen": -1.4210389852523804,
      "logps/rejected": -1.815585732460022,
      "loss": 0.4974,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4210389852523804,
      "rewards/margins": 0.3945468068122864,
      "rewards/rejected": -1.815585732460022,
      "sft_loss": 1.527193307876587,
      "step": 3045
    },
    {
      "epoch": 1.632379996654959,
      "grad_norm": 6.376406445542566,
      "learning_rate": 5.101224257348987e-07,
      "logits/chosen": -0.1394258737564087,
      "logits/rejected": 0.03964899107813835,
      "logps/chosen": -1.5754724740982056,
      "logps/rejected": -2.152021884918213,
      "loss": 0.4591,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5754724740982056,
      "rewards/margins": 0.5765495300292969,
      "rewards/rejected": -2.152021884918213,
      "sft_loss": 1.7064645290374756,
      "step": 3050
    },
    {
      "epoch": 1.6350560294363605,
      "grad_norm": 7.174721499128706,
      "learning_rate": 5.085652956714823e-07,
      "logits/chosen": -0.11442530155181885,
      "logits/rejected": 0.05020016431808472,
      "logps/chosen": -1.5213871002197266,
      "logps/rejected": -2.025872230529785,
      "loss": 0.4664,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5213871002197266,
      "rewards/margins": 0.5044850707054138,
      "rewards/rejected": -2.025872230529785,
      "sft_loss": 1.5640764236450195,
      "step": 3055
    },
    {
      "epoch": 1.6377320622177622,
      "grad_norm": 5.183907088182196,
      "learning_rate": 5.070080825075298e-07,
      "logits/chosen": -0.10387809574604034,
      "logits/rejected": 0.10323204845190048,
      "logps/chosen": -1.549683928489685,
      "logps/rejected": -2.03615403175354,
      "loss": 0.5125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.549683928489685,
      "rewards/margins": 0.48646992444992065,
      "rewards/rejected": -2.03615403175354,
      "sft_loss": 1.6807247400283813,
      "step": 3060
    },
    {
      "epoch": 1.6404080949991637,
      "grad_norm": 9.522880950039765,
      "learning_rate": 5.0545080135113e-07,
      "logits/chosen": 9.657442205934785e-06,
      "logits/rejected": 0.05333443731069565,
      "logps/chosen": -1.5538901090621948,
      "logps/rejected": -2.1399333477020264,
      "loss": 0.4729,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5538901090621948,
      "rewards/margins": 0.5860432386398315,
      "rewards/rejected": -2.1399333477020264,
      "sft_loss": 1.6596462726593018,
      "step": 3065
    },
    {
      "epoch": 1.6430841277805652,
      "grad_norm": 10.083793649113037,
      "learning_rate": 5.038934673110316e-07,
      "logits/chosen": -0.1466716080904007,
      "logits/rejected": -0.003812560345977545,
      "logps/chosen": -1.662615180015564,
      "logps/rejected": -2.175483226776123,
      "loss": 0.5041,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.662615180015564,
      "rewards/margins": 0.51286780834198,
      "rewards/rejected": -2.175483226776123,
      "sft_loss": 1.7367607355117798,
      "step": 3070
    },
    {
      "epoch": 1.645760160561967,
      "grad_norm": 4.758845320619211,
      "learning_rate": 5.023360954964963e-07,
      "logits/chosen": -0.1588786542415619,
      "logits/rejected": -0.0798439010977745,
      "logps/chosen": -1.4433513879776,
      "logps/rejected": -1.896655797958374,
      "loss": 0.4648,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4433513879776,
      "rewards/margins": 0.4533044695854187,
      "rewards/rejected": -1.896655797958374,
      "sft_loss": 1.4855616092681885,
      "step": 3075
    },
    {
      "epoch": 1.6484361933433684,
      "grad_norm": 5.792721475907608,
      "learning_rate": 5.007787010171524e-07,
      "logits/chosen": -0.2315061092376709,
      "logits/rejected": -0.013959243893623352,
      "logps/chosen": -1.4062265157699585,
      "logps/rejected": -1.9291127920150757,
      "loss": 0.4447,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4062265157699585,
      "rewards/margins": 0.5228861570358276,
      "rewards/rejected": -1.9291127920150757,
      "sft_loss": 1.5246238708496094,
      "step": 3080
    },
    {
      "epoch": 1.65111222612477,
      "grad_norm": 4.927542384265485,
      "learning_rate": 4.992212989828477e-07,
      "logits/chosen": -0.01306633185595274,
      "logits/rejected": 0.010603232309222221,
      "logps/chosen": -1.4386698007583618,
      "logps/rejected": -1.898354172706604,
      "loss": 0.4682,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4386698007583618,
      "rewards/margins": 0.4596843719482422,
      "rewards/rejected": -1.898354172706604,
      "sft_loss": 1.5103987455368042,
      "step": 3085
    },
    {
      "epoch": 1.6537882589061716,
      "grad_norm": 6.7826883374939415,
      "learning_rate": 4.976639045035036e-07,
      "logits/chosen": 0.008967900648713112,
      "logits/rejected": 0.10348598659038544,
      "logps/chosen": -1.4943504333496094,
      "logps/rejected": -1.8437782526016235,
      "loss": 0.5393,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4943504333496094,
      "rewards/margins": 0.34942787885665894,
      "rewards/rejected": -1.8437782526016235,
      "sft_loss": 1.626360297203064,
      "step": 3090
    },
    {
      "epoch": 1.6564642916875731,
      "grad_norm": 7.473960780833833,
      "learning_rate": 4.961065326889683e-07,
      "logits/chosen": -0.05825355648994446,
      "logits/rejected": 0.11429224908351898,
      "logps/chosen": -1.5541985034942627,
      "logps/rejected": -2.045668363571167,
      "loss": 0.4772,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5541985034942627,
      "rewards/margins": 0.4914700388908386,
      "rewards/rejected": -2.045668363571167,
      "sft_loss": 1.628761649131775,
      "step": 3095
    },
    {
      "epoch": 1.6591403244689746,
      "grad_norm": 7.821153511440808,
      "learning_rate": 4.9454919864887e-07,
      "logits/chosen": -0.21912486851215363,
      "logits/rejected": -0.06080486252903938,
      "logps/chosen": -1.5123136043548584,
      "logps/rejected": -2.0528147220611572,
      "loss": 0.4691,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5123136043548584,
      "rewards/margins": 0.5405011773109436,
      "rewards/rejected": -2.0528147220611572,
      "sft_loss": 1.6570866107940674,
      "step": 3100
    },
    {
      "epoch": 1.6618163572503764,
      "grad_norm": 8.65323848981444,
      "learning_rate": 4.929919174924701e-07,
      "logits/chosen": -0.13585327565670013,
      "logits/rejected": 0.0843258872628212,
      "logps/chosen": -1.5429242849349976,
      "logps/rejected": -1.9370723962783813,
      "loss": 0.5128,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5429242849349976,
      "rewards/margins": 0.394148051738739,
      "rewards/rejected": -1.9370723962783813,
      "sft_loss": 1.6510931253433228,
      "step": 3105
    },
    {
      "epoch": 1.6644923900317778,
      "grad_norm": 14.129405465063757,
      "learning_rate": 4.914347043285177e-07,
      "logits/chosen": -0.05413060635328293,
      "logits/rejected": 0.0819631814956665,
      "logps/chosen": -1.5078423023223877,
      "logps/rejected": -1.936669111251831,
      "loss": 0.5122,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5078423023223877,
      "rewards/margins": 0.4288269579410553,
      "rewards/rejected": -1.936669111251831,
      "sft_loss": 1.4708707332611084,
      "step": 3110
    },
    {
      "epoch": 1.6671684228131793,
      "grad_norm": 7.656275659256228,
      "learning_rate": 4.898775742651013e-07,
      "logits/chosen": 0.04149172827601433,
      "logits/rejected": 0.15403041243553162,
      "logps/chosen": -1.557957410812378,
      "logps/rejected": -2.077946424484253,
      "loss": 0.4698,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.557957410812378,
      "rewards/margins": 0.519989013671875,
      "rewards/rejected": -2.077946424484253,
      "sft_loss": 1.683276891708374,
      "step": 3115
    },
    {
      "epoch": 1.669844455594581,
      "grad_norm": 7.864028694833735,
      "learning_rate": 4.883205424095037e-07,
      "logits/chosen": -0.12901760637760162,
      "logits/rejected": 0.05095566436648369,
      "logps/chosen": -1.592777132987976,
      "logps/rejected": -2.107862949371338,
      "loss": 0.5214,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.592777132987976,
      "rewards/margins": 0.5150859355926514,
      "rewards/rejected": -2.107862949371338,
      "sft_loss": 1.6193393468856812,
      "step": 3120
    },
    {
      "epoch": 1.6725204883759828,
      "grad_norm": 6.254504666073017,
      "learning_rate": 4.86763623868055e-07,
      "logits/chosen": -0.015704909339547157,
      "logits/rejected": 0.12680071592330933,
      "logps/chosen": -1.5865771770477295,
      "logps/rejected": -2.06243896484375,
      "loss": 0.5038,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5865771770477295,
      "rewards/margins": 0.4758616089820862,
      "rewards/rejected": -2.06243896484375,
      "sft_loss": 1.5720789432525635,
      "step": 3125
    },
    {
      "epoch": 1.675196521157384,
      "grad_norm": 6.19753711179988,
      "learning_rate": 4.852068337459856e-07,
      "logits/chosen": -0.025851398706436157,
      "logits/rejected": 0.16065967082977295,
      "logps/chosen": -1.61211359500885,
      "logps/rejected": -2.105921506881714,
      "loss": 0.473,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.61211359500885,
      "rewards/margins": 0.4938080906867981,
      "rewards/rejected": -2.105921506881714,
      "sft_loss": 1.7118337154388428,
      "step": 3130
    },
    {
      "epoch": 1.6778725539387858,
      "grad_norm": 7.017273961715333,
      "learning_rate": 4.8365018714728e-07,
      "logits/chosen": 0.032778430730104446,
      "logits/rejected": 0.11387556791305542,
      "logps/chosen": -1.614831566810608,
      "logps/rejected": -2.009108066558838,
      "loss": 0.5184,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.614831566810608,
      "rewards/margins": 0.3942764401435852,
      "rewards/rejected": -2.009108066558838,
      "sft_loss": 1.5776565074920654,
      "step": 3135
    },
    {
      "epoch": 1.6805485867201875,
      "grad_norm": 5.893771627132026,
      "learning_rate": 4.820936991745304e-07,
      "logits/chosen": -0.25968000292778015,
      "logits/rejected": -0.08750112354755402,
      "logps/chosen": -1.424504280090332,
      "logps/rejected": -1.8231842517852783,
      "loss": 0.484,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.424504280090332,
      "rewards/margins": 0.3986798822879791,
      "rewards/rejected": -1.8231842517852783,
      "sft_loss": 1.5326725244522095,
      "step": 3140
    },
    {
      "epoch": 1.6832246195015887,
      "grad_norm": 6.211170635152287,
      "learning_rate": 4.8053738492879e-07,
      "logits/chosen": -0.06294619292020798,
      "logits/rejected": 0.10353779792785645,
      "logps/chosen": -1.4389173984527588,
      "logps/rejected": -1.9819748401641846,
      "loss": 0.4727,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4389173984527588,
      "rewards/margins": 0.543057382106781,
      "rewards/rejected": -1.9819748401641846,
      "sft_loss": 1.5338695049285889,
      "step": 3145
    },
    {
      "epoch": 1.6859006522829905,
      "grad_norm": 8.202664843545957,
      "learning_rate": 4.789812595094265e-07,
      "logits/chosen": -0.21335256099700928,
      "logits/rejected": -0.07415179908275604,
      "logps/chosen": -1.5558087825775146,
      "logps/rejected": -2.019449472427368,
      "loss": 0.4847,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5558087825775146,
      "rewards/margins": 0.4636405110359192,
      "rewards/rejected": -2.019449472427368,
      "sft_loss": 1.5690850019454956,
      "step": 3150
    },
    {
      "epoch": 1.6885766850643922,
      "grad_norm": 7.555399006864592,
      "learning_rate": 4.774253380139752e-07,
      "logits/chosen": -0.21016494929790497,
      "logits/rejected": -0.07241862267255783,
      "logps/chosen": -1.3800415992736816,
      "logps/rejected": -1.8963416814804077,
      "loss": 0.4651,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.3800415992736816,
      "rewards/margins": 0.5163003206253052,
      "rewards/rejected": -1.8963416814804077,
      "sft_loss": 1.4642889499664307,
      "step": 3155
    },
    {
      "epoch": 1.6912527178457935,
      "grad_norm": 8.021356820522469,
      "learning_rate": 4.758696355379936e-07,
      "logits/chosen": -0.0793885663151741,
      "logits/rejected": -0.0631255954504013,
      "logps/chosen": -1.495445966720581,
      "logps/rejected": -2.0048792362213135,
      "loss": 0.4772,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.495445966720581,
      "rewards/margins": 0.5094331502914429,
      "rewards/rejected": -2.0048792362213135,
      "sft_loss": 1.6442674398422241,
      "step": 3160
    },
    {
      "epoch": 1.6939287506271952,
      "grad_norm": 5.884767247203592,
      "learning_rate": 4.743141671749138e-07,
      "logits/chosen": -0.21232256293296814,
      "logits/rejected": -0.09611859917640686,
      "logps/chosen": -1.5573443174362183,
      "logps/rejected": -1.9870105981826782,
      "loss": 0.5023,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.5573443174362183,
      "rewards/margins": 0.429666131734848,
      "rewards/rejected": -1.9870105981826782,
      "sft_loss": 1.6721751689910889,
      "step": 3165
    },
    {
      "epoch": 1.6966047834085969,
      "grad_norm": 6.508656695974505,
      "learning_rate": 4.727589480158968e-07,
      "logits/chosen": -0.19365619122982025,
      "logits/rejected": -0.09000679850578308,
      "logps/chosen": -1.5363733768463135,
      "logps/rejected": -2.0378050804138184,
      "loss": 0.487,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5363733768463135,
      "rewards/margins": 0.5014316439628601,
      "rewards/rejected": -2.0378050804138184,
      "sft_loss": 1.603652000427246,
      "step": 3170
    },
    {
      "epoch": 1.6992808161899984,
      "grad_norm": 15.440592748257846,
      "learning_rate": 4.712039931496855e-07,
      "logits/chosen": -0.22241048514842987,
      "logits/rejected": -0.10200734436511993,
      "logps/chosen": -1.5882127285003662,
      "logps/rejected": -1.9402246475219727,
      "loss": 0.5588,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5882127285003662,
      "rewards/margins": 0.35201185941696167,
      "rewards/rejected": -1.9402246475219727,
      "sft_loss": 1.6542675495147705,
      "step": 3175
    },
    {
      "epoch": 1.7019568489713999,
      "grad_norm": 6.565886157189213,
      "learning_rate": 4.6964931766245905e-07,
      "logits/chosen": -0.06484415382146835,
      "logits/rejected": -0.006489972118288279,
      "logps/chosen": -1.5401251316070557,
      "logps/rejected": -2.1260883808135986,
      "loss": 0.4658,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5401251316070557,
      "rewards/margins": 0.5859628915786743,
      "rewards/rejected": -2.1260883808135986,
      "sft_loss": 1.5851103067398071,
      "step": 3180
    },
    {
      "epoch": 1.7046328817528016,
      "grad_norm": 6.891541955713481,
      "learning_rate": 4.6809493663768575e-07,
      "logits/chosen": -0.09114833176136017,
      "logits/rejected": -0.05542738363146782,
      "logps/chosen": -1.5016356706619263,
      "logps/rejected": -1.8026351928710938,
      "loss": 0.5217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5016356706619263,
      "rewards/margins": 0.30099934339523315,
      "rewards/rejected": -1.8026351928710938,
      "sft_loss": 1.6295297145843506,
      "step": 3185
    },
    {
      "epoch": 1.707308914534203,
      "grad_norm": 6.923994823932767,
      "learning_rate": 4.6654086515597716e-07,
      "logits/chosen": -0.19895055890083313,
      "logits/rejected": -0.01881379820406437,
      "logps/chosen": -1.475109338760376,
      "logps/rejected": -2.0406415462493896,
      "loss": 0.4467,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.475109338760376,
      "rewards/margins": 0.5655320882797241,
      "rewards/rejected": -2.0406415462493896,
      "sft_loss": 1.5153456926345825,
      "step": 3190
    },
    {
      "epoch": 1.7099849473156046,
      "grad_norm": 7.188624836614871,
      "learning_rate": 4.6498711829494154e-07,
      "logits/chosen": -0.2148740589618683,
      "logits/rejected": -0.09418477863073349,
      "logps/chosen": -1.4688165187835693,
      "logps/rejected": -1.9643265008926392,
      "loss": 0.4726,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4688165187835693,
      "rewards/margins": 0.4955100417137146,
      "rewards/rejected": -1.9643265008926392,
      "sft_loss": 1.48080575466156,
      "step": 3195
    },
    {
      "epoch": 1.7126609800970063,
      "grad_norm": 8.296804746944462,
      "learning_rate": 4.6343371112903777e-07,
      "logits/chosen": -0.08745540678501129,
      "logits/rejected": 0.06984991580247879,
      "logps/chosen": -1.5411351919174194,
      "logps/rejected": -2.0730738639831543,
      "loss": 0.5053,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5411351919174194,
      "rewards/margins": 0.531938910484314,
      "rewards/rejected": -2.0730738639831543,
      "sft_loss": 1.5940454006195068,
      "step": 3200
    },
    {
      "epoch": 1.7126609800970063,
      "eval_logits/chosen": 0.21331138908863068,
      "eval_logits/rejected": 0.3130818009376526,
      "eval_logps/chosen": -1.5037399530410767,
      "eval_logps/rejected": -1.9595329761505127,
      "eval_loss": 0.49355337023735046,
      "eval_rewards/accuracies": 0.6483679413795471,
      "eval_rewards/chosen": -1.5037399530410767,
      "eval_rewards/margins": 0.455793172121048,
      "eval_rewards/rejected": -1.9595329761505127,
      "eval_runtime": 43.4212,
      "eval_samples_per_second": 30.976,
      "eval_sft_loss": 1.575928807258606,
      "eval_steps_per_second": 7.761,
      "step": 3200
    },
    {
      "epoch": 1.7153370128784078,
      "grad_norm": 7.354230571529315,
      "learning_rate": 4.618806587294291e-07,
      "logits/chosen": -0.2326853722333908,
      "logits/rejected": -0.09422379732131958,
      "logps/chosen": -1.472592830657959,
      "logps/rejected": -1.9556305408477783,
      "loss": 0.4852,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.472592830657959,
      "rewards/margins": 0.4830375611782074,
      "rewards/rejected": -1.9556305408477783,
      "sft_loss": 1.4990339279174805,
      "step": 3205
    },
    {
      "epoch": 1.7180130456598093,
      "grad_norm": 8.17477043186777,
      "learning_rate": 4.603279761638365e-07,
      "logits/chosen": -0.2231977880001068,
      "logits/rejected": -0.11281216144561768,
      "logps/chosen": -1.4896111488342285,
      "logps/rejected": -1.9424409866333008,
      "loss": 0.4781,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.4896111488342285,
      "rewards/margins": 0.4528297781944275,
      "rewards/rejected": -1.9424409866333008,
      "sft_loss": 1.561738133430481,
      "step": 3210
    },
    {
      "epoch": 1.720689078441211,
      "grad_norm": 7.828530935557396,
      "learning_rate": 4.5877567849639315e-07,
      "logits/chosen": -0.18882520496845245,
      "logits/rejected": -0.049553271383047104,
      "logps/chosen": -1.467195749282837,
      "logps/rejected": -1.9305130243301392,
      "loss": 0.4946,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.467195749282837,
      "rewards/margins": 0.4633170962333679,
      "rewards/rejected": -1.9305130243301392,
      "sft_loss": 1.5044894218444824,
      "step": 3215
    },
    {
      "epoch": 1.7233651112226125,
      "grad_norm": 6.379942477646264,
      "learning_rate": 4.572237807874979e-07,
      "logits/chosen": -0.21386948227882385,
      "logits/rejected": 0.03034193255007267,
      "logps/chosen": -1.6358089447021484,
      "logps/rejected": -2.112433910369873,
      "loss": 0.5479,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.6358089447021484,
      "rewards/margins": 0.4766252636909485,
      "rewards/rejected": -2.112433910369873,
      "sft_loss": 1.6175771951675415,
      "step": 3220
    },
    {
      "epoch": 1.726041144004014,
      "grad_norm": 7.627844508547667,
      "learning_rate": 4.5567229809366895e-07,
      "logits/chosen": -0.1952795684337616,
      "logits/rejected": -0.056179117411375046,
      "logps/chosen": -1.3949609994888306,
      "logps/rejected": -1.951027274131775,
      "loss": 0.446,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3949609994888306,
      "rewards/margins": 0.5560663342475891,
      "rewards/rejected": -1.951027274131775,
      "sft_loss": 1.4932529926300049,
      "step": 3225
    },
    {
      "epoch": 1.7287171767854157,
      "grad_norm": 8.166108560103117,
      "learning_rate": 4.541212454673984e-07,
      "logits/chosen": -0.20898957550525665,
      "logits/rejected": -0.03555748611688614,
      "logps/chosen": -1.4980252981185913,
      "logps/rejected": -2.201258420944214,
      "loss": 0.4252,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4980252981185913,
      "rewards/margins": 0.7032328844070435,
      "rewards/rejected": -2.201258420944214,
      "sft_loss": 1.5434107780456543,
      "step": 3230
    },
    {
      "epoch": 1.7313932095668172,
      "grad_norm": 6.480679064846322,
      "learning_rate": 4.525706379570055e-07,
      "logits/chosen": -0.18776097893714905,
      "logits/rejected": -0.10159311443567276,
      "logps/chosen": -1.5042344331741333,
      "logps/rejected": -2.028208017349243,
      "loss": 0.4707,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5042344331741333,
      "rewards/margins": 0.5239735841751099,
      "rewards/rejected": -2.028208017349243,
      "sft_loss": 1.5815365314483643,
      "step": 3235
    },
    {
      "epoch": 1.7340692423482187,
      "grad_norm": 7.34266127543648,
      "learning_rate": 4.510204906064911e-07,
      "logits/chosen": -0.10263445228338242,
      "logits/rejected": 0.02675381675362587,
      "logps/chosen": -1.4704691171646118,
      "logps/rejected": -2.0698206424713135,
      "loss": 0.4565,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4704691171646118,
      "rewards/margins": 0.5993512272834778,
      "rewards/rejected": -2.0698206424713135,
      "sft_loss": 1.4575080871582031,
      "step": 3240
    },
    {
      "epoch": 1.7367452751296204,
      "grad_norm": 11.529074535047988,
      "learning_rate": 4.4947081845539177e-07,
      "logits/chosen": -0.2872398793697357,
      "logits/rejected": -0.128311425447464,
      "logps/chosen": -1.5303713083267212,
      "logps/rejected": -2.069119691848755,
      "loss": 0.482,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5303713083267212,
      "rewards/margins": 0.5387482643127441,
      "rewards/rejected": -2.069119691848755,
      "sft_loss": 1.528638243675232,
      "step": 3245
    },
    {
      "epoch": 1.739421307911022,
      "grad_norm": 6.229660340308704,
      "learning_rate": 4.479216365386333e-07,
      "logits/chosen": -0.08383894711732864,
      "logits/rejected": 0.07514307647943497,
      "logps/chosen": -1.5344181060791016,
      "logps/rejected": -2.0951080322265625,
      "loss": 0.4722,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5344181060791016,
      "rewards/margins": 0.5606899857521057,
      "rewards/rejected": -2.0951080322265625,
      "sft_loss": 1.594663381576538,
      "step": 3250
    },
    {
      "epoch": 1.7420973406924234,
      "grad_norm": 5.422422697510295,
      "learning_rate": 4.4637295988638555e-07,
      "logits/chosen": -0.10217050462961197,
      "logits/rejected": -0.02202165499329567,
      "logps/chosen": -1.5932085514068604,
      "logps/rejected": -1.9968818426132202,
      "loss": 0.5188,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5932085514068604,
      "rewards/margins": 0.4036734104156494,
      "rewards/rejected": -1.9968818426132202,
      "sft_loss": 1.6095945835113525,
      "step": 3255
    },
    {
      "epoch": 1.744773373473825,
      "grad_norm": 8.694436413759197,
      "learning_rate": 4.4482480352391623e-07,
      "logits/chosen": -0.24071171879768372,
      "logits/rejected": -0.08918756991624832,
      "logps/chosen": -1.5446043014526367,
      "logps/rejected": -2.0271828174591064,
      "loss": 0.4833,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5446043014526367,
      "rewards/margins": 0.4825782775878906,
      "rewards/rejected": -2.0271828174591064,
      "sft_loss": 1.616150140762329,
      "step": 3260
    },
    {
      "epoch": 1.7474494062552266,
      "grad_norm": 9.264782264435171,
      "learning_rate": 4.4327718247144507e-07,
      "logits/chosen": -0.12294987589120865,
      "logits/rejected": -0.005746960639953613,
      "logps/chosen": -1.4709341526031494,
      "logps/rejected": -2.0256457328796387,
      "loss": 0.4702,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4709341526031494,
      "rewards/margins": 0.5547112822532654,
      "rewards/rejected": -2.0256457328796387,
      "sft_loss": 1.5738205909729004,
      "step": 3265
    },
    {
      "epoch": 1.750125439036628,
      "grad_norm": 9.243662501602794,
      "learning_rate": 4.417301117439984e-07,
      "logits/chosen": -0.13053932785987854,
      "logits/rejected": 0.021624181419610977,
      "logps/chosen": -1.4685636758804321,
      "logps/rejected": -2.024125337600708,
      "loss": 0.4577,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4685636758804321,
      "rewards/margins": 0.5555616617202759,
      "rewards/rejected": -2.024125337600708,
      "sft_loss": 1.533262848854065,
      "step": 3270
    },
    {
      "epoch": 1.7528014718180298,
      "grad_norm": 6.070545992868571,
      "learning_rate": 4.401836063512631e-07,
      "logits/chosen": -0.2038416564464569,
      "logits/rejected": 0.12645886838436127,
      "logps/chosen": -1.5461547374725342,
      "logps/rejected": -2.1197752952575684,
      "loss": 0.4589,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5461547374725342,
      "rewards/margins": 0.5736207962036133,
      "rewards/rejected": -2.1197752952575684,
      "sft_loss": 1.6438357830047607,
      "step": 3275
    },
    {
      "epoch": 1.7554775045994313,
      "grad_norm": 9.534324648344255,
      "learning_rate": 4.386376812974413e-07,
      "logits/chosen": -0.16504624485969543,
      "logits/rejected": -0.058779411017894745,
      "logps/chosen": -1.3824862241744995,
      "logps/rejected": -1.8809837102890015,
      "loss": 0.4524,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3824862241744995,
      "rewards/margins": 0.4984973073005676,
      "rewards/rejected": -1.8809837102890015,
      "sft_loss": 1.5216165781021118,
      "step": 3280
    },
    {
      "epoch": 1.7581535373808328,
      "grad_norm": 6.194222891691346,
      "learning_rate": 4.370923515811048e-07,
      "logits/chosen": -0.19524803757667542,
      "logits/rejected": 0.04105467349290848,
      "logps/chosen": -1.4834949970245361,
      "logps/rejected": -2.0216991901397705,
      "loss": 0.4608,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4834949970245361,
      "rewards/margins": 0.5382041931152344,
      "rewards/rejected": -2.0216991901397705,
      "sft_loss": 1.5563383102416992,
      "step": 3285
    },
    {
      "epoch": 1.7608295701622345,
      "grad_norm": 8.985288741061424,
      "learning_rate": 4.35547632195049e-07,
      "logits/chosen": -0.11997727304697037,
      "logits/rejected": 0.0095209376886487,
      "logps/chosen": -1.4815948009490967,
      "logps/rejected": -1.9493820667266846,
      "loss": 0.482,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4815948009490967,
      "rewards/margins": 0.4677874445915222,
      "rewards/rejected": -1.9493820667266846,
      "sft_loss": 1.5820047855377197,
      "step": 3290
    },
    {
      "epoch": 1.763505602943636,
      "grad_norm": 6.636082849373608,
      "learning_rate": 4.340035381261484e-07,
      "logits/chosen": -0.159527987241745,
      "logits/rejected": -0.08030469715595245,
      "logps/chosen": -1.659623146057129,
      "logps/rejected": -2.1283786296844482,
      "loss": 0.5277,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.659623146057129,
      "rewards/margins": 0.4687553942203522,
      "rewards/rejected": -2.1283786296844482,
      "sft_loss": 1.684664011001587,
      "step": 3295
    },
    {
      "epoch": 1.7661816357250375,
      "grad_norm": 9.20108436395257,
      "learning_rate": 4.324600843552104e-07,
      "logits/chosen": -0.28877630829811096,
      "logits/rejected": -0.13612669706344604,
      "logps/chosen": -1.6840623617172241,
      "logps/rejected": -2.2746193408966064,
      "loss": 0.4984,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.6840623617172241,
      "rewards/margins": 0.5905566811561584,
      "rewards/rejected": -2.2746193408966064,
      "sft_loss": 1.7702968120574951,
      "step": 3300
    },
    {
      "epoch": 1.7688576685064392,
      "grad_norm": 9.077056353232631,
      "learning_rate": 4.309172858568302e-07,
      "logits/chosen": -0.24776725471019745,
      "logits/rejected": -0.07644428312778473,
      "logps/chosen": -1.6167583465576172,
      "logps/rejected": -2.170414447784424,
      "loss": 0.4828,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6167583465576172,
      "rewards/margins": 0.553656280040741,
      "rewards/rejected": -2.170414447784424,
      "sft_loss": 1.6717586517333984,
      "step": 3305
    },
    {
      "epoch": 1.771533701287841,
      "grad_norm": 6.04205711769031,
      "learning_rate": 4.293751575992455e-07,
      "logits/chosen": -0.03785474970936775,
      "logits/rejected": 0.021119682118296623,
      "logps/chosen": -1.5902318954467773,
      "logps/rejected": -2.062140941619873,
      "loss": 0.4824,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5902318954467773,
      "rewards/margins": 0.47190895676612854,
      "rewards/rejected": -2.062140941619873,
      "sft_loss": 1.6707704067230225,
      "step": 3310
    },
    {
      "epoch": 1.7742097340692422,
      "grad_norm": 11.243449946581617,
      "learning_rate": 4.278337145441916e-07,
      "logits/chosen": -0.2524152398109436,
      "logits/rejected": -0.07547532767057419,
      "logps/chosen": -1.5760393142700195,
      "logps/rejected": -2.067971706390381,
      "loss": 0.4972,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5760393142700195,
      "rewards/margins": 0.49193257093429565,
      "rewards/rejected": -2.067971706390381,
      "sft_loss": 1.6471372842788696,
      "step": 3315
    },
    {
      "epoch": 1.776885766850644,
      "grad_norm": 6.114760882832403,
      "learning_rate": 4.262929716467556e-07,
      "logits/chosen": -0.13797271251678467,
      "logits/rejected": 0.08544354140758514,
      "logps/chosen": -1.5435459613800049,
      "logps/rejected": -2.251192569732666,
      "loss": 0.4458,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5435459613800049,
      "rewards/margins": 0.7076464891433716,
      "rewards/rejected": -2.251192569732666,
      "sft_loss": 1.643958330154419,
      "step": 3320
    },
    {
      "epoch": 1.7795617996320456,
      "grad_norm": 7.098971135859354,
      "learning_rate": 4.247529438552321e-07,
      "logits/chosen": -0.21761508285999298,
      "logits/rejected": -0.0269846860319376,
      "logps/chosen": -1.5963890552520752,
      "logps/rejected": -2.1412088871002197,
      "loss": 0.4671,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5963890552520752,
      "rewards/margins": 0.5448198318481445,
      "rewards/rejected": -2.1412088871002197,
      "sft_loss": 1.7654186487197876,
      "step": 3325
    },
    {
      "epoch": 1.782237832413447,
      "grad_norm": 7.508412069749556,
      "learning_rate": 4.232136461109773e-07,
      "logits/chosen": -0.09692569822072983,
      "logits/rejected": 0.03345852345228195,
      "logps/chosen": -1.4379321336746216,
      "logps/rejected": -2.0407073497772217,
      "loss": 0.4442,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4379321336746216,
      "rewards/margins": 0.6027752757072449,
      "rewards/rejected": -2.0407073497772217,
      "sft_loss": 1.5631394386291504,
      "step": 3330
    },
    {
      "epoch": 1.7849138651948486,
      "grad_norm": 23.373034792821425,
      "learning_rate": 4.216750933482646e-07,
      "logits/chosen": -0.1263446807861328,
      "logits/rejected": 0.04607892036437988,
      "logps/chosen": -1.598233699798584,
      "logps/rejected": -2.06453275680542,
      "loss": 0.5065,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.598233699798584,
      "rewards/margins": 0.4662991464138031,
      "rewards/rejected": -2.06453275680542,
      "sft_loss": 1.600513219833374,
      "step": 3335
    },
    {
      "epoch": 1.7875898979762503,
      "grad_norm": 6.915232794749861,
      "learning_rate": 4.2013730049413986e-07,
      "logits/chosen": -0.09540759027004242,
      "logits/rejected": 0.08306506276130676,
      "logps/chosen": -1.4474116563796997,
      "logps/rejected": -1.98406982421875,
      "loss": 0.4733,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4474116563796997,
      "rewards/margins": 0.5366581082344055,
      "rewards/rejected": -1.98406982421875,
      "sft_loss": 1.5605659484863281,
      "step": 3340
    },
    {
      "epoch": 1.7902659307576518,
      "grad_norm": 6.827983381630936,
      "learning_rate": 4.1860028246827594e-07,
      "logits/chosen": -0.12237314134836197,
      "logits/rejected": 0.07193153351545334,
      "logps/chosen": -1.3919322490692139,
      "logps/rejected": -1.8691747188568115,
      "loss": 0.4674,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.3919322490692139,
      "rewards/margins": 0.4772423803806305,
      "rewards/rejected": -1.8691747188568115,
      "sft_loss": 1.5088390111923218,
      "step": 3345
    },
    {
      "epoch": 1.7929419635390533,
      "grad_norm": 8.547269408446702,
      "learning_rate": 4.170640541828285e-07,
      "logits/chosen": -0.28706783056259155,
      "logits/rejected": -0.12526081502437592,
      "logps/chosen": -1.5753610134124756,
      "logps/rejected": -1.9887001514434814,
      "loss": 0.496,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5753610134124756,
      "rewards/margins": 0.4133389890193939,
      "rewards/rejected": -1.9887001514434814,
      "sft_loss": 1.6392730474472046,
      "step": 3350
    },
    {
      "epoch": 1.795617996320455,
      "grad_norm": 6.253426926775852,
      "learning_rate": 4.1552863054229116e-07,
      "logits/chosen": -0.02050682343542576,
      "logits/rejected": 0.031059980392456055,
      "logps/chosen": -1.6005933284759521,
      "logps/rejected": -1.961517095565796,
      "loss": 0.5675,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.6005933284759521,
      "rewards/margins": 0.3609238266944885,
      "rewards/rejected": -1.961517095565796,
      "sft_loss": 1.6103566884994507,
      "step": 3355
    },
    {
      "epoch": 1.7982940291018565,
      "grad_norm": 7.896506850980646,
      "learning_rate": 4.139940264433508e-07,
      "logits/chosen": -0.1889674961566925,
      "logits/rejected": 0.057184554636478424,
      "logps/chosen": -1.4576613903045654,
      "logps/rejected": -1.9298509359359741,
      "loss": 0.4801,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4576613903045654,
      "rewards/margins": 0.4721898138523102,
      "rewards/rejected": -1.9298509359359741,
      "sft_loss": 1.5358812808990479,
      "step": 3360
    },
    {
      "epoch": 1.800970061883258,
      "grad_norm": 5.739958509027284,
      "learning_rate": 4.1246025677474303e-07,
      "logits/chosen": -0.18853381276130676,
      "logits/rejected": -0.010739481076598167,
      "logps/chosen": -1.5681513547897339,
      "logps/rejected": -2.059718608856201,
      "loss": 0.4769,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5681513547897339,
      "rewards/margins": 0.4915672242641449,
      "rewards/rejected": -2.059718608856201,
      "sft_loss": 1.6615225076675415,
      "step": 3365
    },
    {
      "epoch": 1.8036460946646597,
      "grad_norm": 9.963203520040665,
      "learning_rate": 4.10927336417108e-07,
      "logits/chosen": -0.19930417835712433,
      "logits/rejected": -0.023639291524887085,
      "logps/chosen": -1.5441948175430298,
      "logps/rejected": -1.9339659214019775,
      "loss": 0.5351,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5441948175430298,
      "rewards/margins": 0.389771044254303,
      "rewards/rejected": -1.9339659214019775,
      "sft_loss": 1.636212706565857,
      "step": 3370
    },
    {
      "epoch": 1.8063221274460612,
      "grad_norm": 10.36044116532147,
      "learning_rate": 4.093952802428457e-07,
      "logits/chosen": -0.03929683566093445,
      "logits/rejected": 0.030060648918151855,
      "logps/chosen": -1.638843297958374,
      "logps/rejected": -2.066988468170166,
      "loss": 0.5512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.638843297958374,
      "rewards/margins": 0.42814522981643677,
      "rewards/rejected": -2.066988468170166,
      "sft_loss": 1.6774746179580688,
      "step": 3375
    },
    {
      "epoch": 1.8089981602274627,
      "grad_norm": 5.500783193468918,
      "learning_rate": 4.0786410311597184e-07,
      "logits/chosen": -0.23853763937950134,
      "logits/rejected": -0.06972865760326385,
      "logps/chosen": -1.5138180255889893,
      "logps/rejected": -2.024914026260376,
      "loss": 0.4648,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5138180255889893,
      "rewards/margins": 0.5110958218574524,
      "rewards/rejected": -2.024914026260376,
      "sft_loss": 1.5387523174285889,
      "step": 3380
    },
    {
      "epoch": 1.8116741930088645,
      "grad_norm": 5.7217239160253,
      "learning_rate": 4.063338198919737e-07,
      "logits/chosen": -0.21125145256519318,
      "logits/rejected": -0.17624393105506897,
      "logps/chosen": -1.6046793460845947,
      "logps/rejected": -1.986384630203247,
      "loss": 0.5189,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.6046793460845947,
      "rewards/margins": 0.3817053735256195,
      "rewards/rejected": -1.986384630203247,
      "sft_loss": 1.6781991720199585,
      "step": 3385
    },
    {
      "epoch": 1.814350225790266,
      "grad_norm": 9.763318573444817,
      "learning_rate": 4.0480444541766575e-07,
      "logits/chosen": -0.15552251040935516,
      "logits/rejected": -0.016761211678385735,
      "logps/chosen": -1.5631591081619263,
      "logps/rejected": -1.9353306293487549,
      "loss": 0.5276,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5631591081619263,
      "rewards/margins": 0.3721713423728943,
      "rewards/rejected": -1.9353306293487549,
      "sft_loss": 1.6414425373077393,
      "step": 3390
    },
    {
      "epoch": 1.8170262585716674,
      "grad_norm": 8.200539388456589,
      "learning_rate": 4.0327599453104606e-07,
      "logits/chosen": -0.18579599261283875,
      "logits/rejected": -0.06101926043629646,
      "logps/chosen": -1.429839849472046,
      "logps/rejected": -1.963600754737854,
      "loss": 0.4626,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.429839849472046,
      "rewards/margins": 0.5337609052658081,
      "rewards/rejected": -1.963600754737854,
      "sft_loss": 1.4940623044967651,
      "step": 3395
    },
    {
      "epoch": 1.8197022913530692,
      "grad_norm": 8.778522639839275,
      "learning_rate": 4.017484820611514e-07,
      "logits/chosen": -0.15336444973945618,
      "logits/rejected": -0.025536376982927322,
      "logps/chosen": -1.4731427431106567,
      "logps/rejected": -1.906964898109436,
      "loss": 0.5016,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.4731427431106567,
      "rewards/margins": 0.4338221549987793,
      "rewards/rejected": -1.906964898109436,
      "sft_loss": 1.5429713726043701,
      "step": 3400
    },
    {
      "epoch": 1.8223783241344707,
      "grad_norm": 6.1224910256530745,
      "learning_rate": 4.002219228279148e-07,
      "logits/chosen": -0.15219996869564056,
      "logits/rejected": 0.021372251212596893,
      "logps/chosen": -1.4793925285339355,
      "logps/rejected": -1.9215284585952759,
      "loss": 0.4807,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4793925285339355,
      "rewards/margins": 0.44213590025901794,
      "rewards/rejected": -1.9215284585952759,
      "sft_loss": 1.5823194980621338,
      "step": 3405
    },
    {
      "epoch": 1.8250543569158721,
      "grad_norm": 11.156015775060384,
      "learning_rate": 3.9869633164202045e-07,
      "logits/chosen": -0.16902689635753632,
      "logits/rejected": 0.09002505242824554,
      "logps/chosen": -1.6346511840820312,
      "logps/rejected": -2.040658473968506,
      "loss": 0.5144,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.6346511840820312,
      "rewards/margins": 0.4060073792934418,
      "rewards/rejected": -2.040658473968506,
      "sft_loss": 1.6519008874893188,
      "step": 3410
    },
    {
      "epoch": 1.8277303896972739,
      "grad_norm": 11.681531329961869,
      "learning_rate": 3.9717172330476077e-07,
      "logits/chosen": -0.1769445389509201,
      "logits/rejected": -0.048740632832050323,
      "logps/chosen": -1.5493731498718262,
      "logps/rejected": -2.083101272583008,
      "loss": 0.49,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5493731498718262,
      "rewards/margins": 0.5337280631065369,
      "rewards/rejected": -2.083101272583008,
      "sft_loss": 1.6823348999023438,
      "step": 3415
    },
    {
      "epoch": 1.8304064224786754,
      "grad_norm": 12.745222763832293,
      "learning_rate": 3.956481126078927e-07,
      "logits/chosen": -0.1154409795999527,
      "logits/rejected": 0.015516482293605804,
      "logps/chosen": -1.614915132522583,
      "logps/rejected": -2.193376302719116,
      "loss": 0.5011,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.614915132522583,
      "rewards/margins": 0.5784613490104675,
      "rewards/rejected": -2.193376302719116,
      "sft_loss": 1.7101062536239624,
      "step": 3420
    },
    {
      "epoch": 1.8330824552600768,
      "grad_norm": 5.527779575795557,
      "learning_rate": 3.941255143334937e-07,
      "logits/chosen": -0.1938527524471283,
      "logits/rejected": -0.1387048065662384,
      "logps/chosen": -1.4739712476730347,
      "logps/rejected": -1.9787203073501587,
      "loss": 0.4888,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4739712476730347,
      "rewards/margins": 0.5047491192817688,
      "rewards/rejected": -1.9787203073501587,
      "sft_loss": 1.5416609048843384,
      "step": 3425
    },
    {
      "epoch": 1.8357584880414786,
      "grad_norm": 6.6023513847062,
      "learning_rate": 3.9260394325381895e-07,
      "logits/chosen": -0.19061097502708435,
      "logits/rejected": -0.047849975526332855,
      "logps/chosen": -1.5689013004302979,
      "logps/rejected": -2.236057758331299,
      "loss": 0.4749,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5689013004302979,
      "rewards/margins": 0.6671566367149353,
      "rewards/rejected": -2.236057758331299,
      "sft_loss": 1.6300235986709595,
      "step": 3430
    },
    {
      "epoch": 1.83843452082288,
      "grad_norm": 7.943540586218268,
      "learning_rate": 3.9108341413115784e-07,
      "logits/chosen": -0.18472620844841003,
      "logits/rejected": -0.07400848716497421,
      "logps/chosen": -1.4624192714691162,
      "logps/rejected": -2.032019853591919,
      "loss": 0.436,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4624192714691162,
      "rewards/margins": 0.569600522518158,
      "rewards/rejected": -2.032019853591919,
      "sft_loss": 1.5836392641067505,
      "step": 3435
    },
    {
      "epoch": 1.8411105536042816,
      "grad_norm": 8.677362616663258,
      "learning_rate": 3.895639417176905e-07,
      "logits/chosen": -0.21190118789672852,
      "logits/rejected": -0.13000845909118652,
      "logps/chosen": -1.41072678565979,
      "logps/rejected": -2.0230135917663574,
      "loss": 0.4718,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.41072678565979,
      "rewards/margins": 0.6122865676879883,
      "rewards/rejected": -2.0230135917663574,
      "sft_loss": 1.473109245300293,
      "step": 3440
    },
    {
      "epoch": 1.8437865863856833,
      "grad_norm": 5.332106728284266,
      "learning_rate": 3.8804554075534497e-07,
      "logits/chosen": -0.21277959644794464,
      "logits/rejected": 0.03557942062616348,
      "logps/chosen": -1.5381152629852295,
      "logps/rejected": -2.060868740081787,
      "loss": 0.4841,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5381152629852295,
      "rewards/margins": 0.5227535367012024,
      "rewards/rejected": -2.060868740081787,
      "sft_loss": 1.616860032081604,
      "step": 3445
    },
    {
      "epoch": 1.8464626191670848,
      "grad_norm": 8.743005708525434,
      "learning_rate": 3.8652822597565403e-07,
      "logits/chosen": -0.31704509258270264,
      "logits/rejected": -0.11927709728479385,
      "logps/chosen": -1.5102241039276123,
      "logps/rejected": -2.0983102321624756,
      "loss": 0.465,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5102241039276123,
      "rewards/margins": 0.5880860090255737,
      "rewards/rejected": -2.0983102321624756,
      "sft_loss": 1.6415538787841797,
      "step": 3450
    },
    {
      "epoch": 1.8491386519484863,
      "grad_norm": 6.495625379078563,
      "learning_rate": 3.850120120996123e-07,
      "logits/chosen": -0.15546521544456482,
      "logits/rejected": 0.04172942787408829,
      "logps/chosen": -1.7578779458999634,
      "logps/rejected": -2.3052031993865967,
      "loss": 0.5154,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7578779458999634,
      "rewards/margins": 0.5473250150680542,
      "rewards/rejected": -2.3052031993865967,
      "sft_loss": 1.8285331726074219,
      "step": 3455
    },
    {
      "epoch": 1.851814684729888,
      "grad_norm": 9.959675323722992,
      "learning_rate": 3.8349691383753356e-07,
      "logits/chosen": -0.04424437880516052,
      "logits/rejected": 0.09680449217557907,
      "logps/chosen": -1.5204256772994995,
      "logps/rejected": -2.0713820457458496,
      "loss": 0.4881,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5204256772994995,
      "rewards/margins": 0.5509562492370605,
      "rewards/rejected": -2.0713820457458496,
      "sft_loss": 1.589459776878357,
      "step": 3460
    },
    {
      "epoch": 1.8544907175112895,
      "grad_norm": 6.20212722185588,
      "learning_rate": 3.819829458889078e-07,
      "logits/chosen": -0.20593221485614777,
      "logits/rejected": -0.0690714567899704,
      "logps/chosen": -1.4725421667099,
      "logps/rejected": -1.945373773574829,
      "loss": 0.4589,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4725421667099,
      "rewards/margins": 0.47283148765563965,
      "rewards/rejected": -1.945373773574829,
      "sft_loss": 1.5715610980987549,
      "step": 3465
    },
    {
      "epoch": 1.857166750292691,
      "grad_norm": 8.217769659561885,
      "learning_rate": 3.804701229422585e-07,
      "logits/chosen": -0.23379914462566376,
      "logits/rejected": -0.13025447726249695,
      "logps/chosen": -1.6423708200454712,
      "logps/rejected": -2.1803269386291504,
      "loss": 0.4927,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.6423708200454712,
      "rewards/margins": 0.5379562973976135,
      "rewards/rejected": -2.1803269386291504,
      "sft_loss": 1.752219796180725,
      "step": 3470
    },
    {
      "epoch": 1.8598427830740927,
      "grad_norm": 12.838511306887499,
      "learning_rate": 3.789584596750007e-07,
      "logits/chosen": -0.22167137265205383,
      "logits/rejected": -0.14777874946594238,
      "logps/chosen": -1.5322400331497192,
      "logps/rejected": -1.9573066234588623,
      "loss": 0.4892,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5322400331497192,
      "rewards/margins": 0.4250665605068207,
      "rewards/rejected": -1.9573066234588623,
      "sft_loss": 1.5717178583145142,
      "step": 3475
    },
    {
      "epoch": 1.8625188158554944,
      "grad_norm": 5.921295803022303,
      "learning_rate": 3.77447970753298e-07,
      "logits/chosen": -0.06847834587097168,
      "logits/rejected": -0.023401662707328796,
      "logps/chosen": -1.6123988628387451,
      "logps/rejected": -2.10270619392395,
      "loss": 0.4923,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.6123988628387451,
      "rewards/margins": 0.4903070330619812,
      "rewards/rejected": -2.10270619392395,
      "sft_loss": 1.6930005550384521,
      "step": 3480
    },
    {
      "epoch": 1.8651948486368957,
      "grad_norm": 5.80656323015529,
      "learning_rate": 3.7593867083192057e-07,
      "logits/chosen": -0.11366887390613556,
      "logits/rejected": 0.013406882993876934,
      "logps/chosen": -1.5091646909713745,
      "logps/rejected": -1.9836050271987915,
      "loss": 0.4801,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5091646909713745,
      "rewards/margins": 0.4744402766227722,
      "rewards/rejected": -1.9836050271987915,
      "sft_loss": 1.6238653659820557,
      "step": 3485
    },
    {
      "epoch": 1.8678708814182974,
      "grad_norm": 8.682702224715547,
      "learning_rate": 3.7443057455410276e-07,
      "logits/chosen": -0.11430926620960236,
      "logits/rejected": 0.03476153686642647,
      "logps/chosen": -1.548940658569336,
      "logps/rejected": -1.9343265295028687,
      "loss": 0.4977,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.548940658569336,
      "rewards/margins": 0.3853859603404999,
      "rewards/rejected": -1.9343265295028687,
      "sft_loss": 1.6631542444229126,
      "step": 3490
    },
    {
      "epoch": 1.870546914199699,
      "grad_norm": 8.461362933358332,
      "learning_rate": 3.7292369655140145e-07,
      "logits/chosen": -0.2042984962463379,
      "logits/rejected": -0.011971579864621162,
      "logps/chosen": -1.618351697921753,
      "logps/rejected": -2.0287108421325684,
      "loss": 0.4896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.618351697921753,
      "rewards/margins": 0.41035908460617065,
      "rewards/rejected": -2.0287108421325684,
      "sft_loss": 1.7515051364898682,
      "step": 3495
    },
    {
      "epoch": 1.8732229469811004,
      "grad_norm": 5.96404845236303,
      "learning_rate": 3.714180514435534e-07,
      "logits/chosen": -0.11476609855890274,
      "logits/rejected": 0.05753784254193306,
      "logps/chosen": -1.5358325242996216,
      "logps/rejected": -2.1429457664489746,
      "loss": 0.4733,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5358325242996216,
      "rewards/margins": 0.6071134805679321,
      "rewards/rejected": -2.1429457664489746,
      "sft_loss": 1.6601890325546265,
      "step": 3500
    },
    {
      "epoch": 1.875898979762502,
      "grad_norm": 10.451011548578464,
      "learning_rate": 3.6991365383833426e-07,
      "logits/chosen": -0.13792508840560913,
      "logits/rejected": 0.020068520680069923,
      "logps/chosen": -1.5533324480056763,
      "logps/rejected": -2.105034351348877,
      "loss": 0.47,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5533324480056763,
      "rewards/margins": 0.5517022609710693,
      "rewards/rejected": -2.105034351348877,
      "sft_loss": 1.6532182693481445,
      "step": 3505
    },
    {
      "epoch": 1.8785750125439038,
      "grad_norm": 10.590718385432426,
      "learning_rate": 3.684105183314162e-07,
      "logits/chosen": -0.1606968343257904,
      "logits/rejected": -0.0639537051320076,
      "logps/chosen": -1.4895262718200684,
      "logps/rejected": -2.011099100112915,
      "loss": 0.4611,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4895262718200684,
      "rewards/margins": 0.5215727090835571,
      "rewards/rejected": -2.011099100112915,
      "sft_loss": 1.5524523258209229,
      "step": 3510
    },
    {
      "epoch": 1.881251045325305,
      "grad_norm": 7.410572086527162,
      "learning_rate": 3.669086595062263e-07,
      "logits/chosen": -0.17349793016910553,
      "logits/rejected": 0.04328025132417679,
      "logps/chosen": -1.5520845651626587,
      "logps/rejected": -2.1199307441711426,
      "loss": 0.4797,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5520845651626587,
      "rewards/margins": 0.5678460001945496,
      "rewards/rejected": -2.1199307441711426,
      "sft_loss": 1.5678256750106812,
      "step": 3515
    },
    {
      "epoch": 1.8839270781067068,
      "grad_norm": 6.852317477346541,
      "learning_rate": 3.654080919338056e-07,
      "logits/chosen": -0.22616633772850037,
      "logits/rejected": -0.058461517095565796,
      "logps/chosen": -1.5593931674957275,
      "logps/rejected": -2.1259655952453613,
      "loss": 0.4745,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5593931674957275,
      "rewards/margins": 0.5665722489356995,
      "rewards/rejected": -2.1259655952453613,
      "sft_loss": 1.6664631366729736,
      "step": 3520
    },
    {
      "epoch": 1.8866031108881085,
      "grad_norm": 6.478689285386118,
      "learning_rate": 3.639088301726673e-07,
      "logits/chosen": -0.12069617211818695,
      "logits/rejected": 0.09780598431825638,
      "logps/chosen": -1.5308897495269775,
      "logps/rejected": -2.0637001991271973,
      "loss": 0.4709,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5308897495269775,
      "rewards/margins": 0.5328103303909302,
      "rewards/rejected": -2.0637001991271973,
      "sft_loss": 1.6409518718719482,
      "step": 3525
    },
    {
      "epoch": 1.88927914366951,
      "grad_norm": 8.604893947210467,
      "learning_rate": 3.624108887686556e-07,
      "logits/chosen": -0.10809512436389923,
      "logits/rejected": -0.02255006693303585,
      "logps/chosen": -1.5483064651489258,
      "logps/rejected": -2.0952706336975098,
      "loss": 0.466,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5483064651489258,
      "rewards/margins": 0.5469641089439392,
      "rewards/rejected": -2.0952706336975098,
      "sft_loss": 1.6761009693145752,
      "step": 3530
    },
    {
      "epoch": 1.8919551764509115,
      "grad_norm": 9.206865932072462,
      "learning_rate": 3.6091428225480433e-07,
      "logits/chosen": -0.23768194019794464,
      "logits/rejected": -0.08524081856012344,
      "logps/chosen": -1.4865976572036743,
      "logps/rejected": -2.0263915061950684,
      "loss": 0.4689,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.4865976572036743,
      "rewards/margins": 0.5397937893867493,
      "rewards/rejected": -2.0263915061950684,
      "sft_loss": 1.6183054447174072,
      "step": 3535
    },
    {
      "epoch": 1.8946312092323132,
      "grad_norm": 10.922365841022103,
      "learning_rate": 3.5941902515119674e-07,
      "logits/chosen": -0.2043650597333908,
      "logits/rejected": 0.04252279922366142,
      "logps/chosen": -1.570642113685608,
      "logps/rejected": -1.981335997581482,
      "loss": 0.5105,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.570642113685608,
      "rewards/margins": 0.4106937348842621,
      "rewards/rejected": -1.981335997581482,
      "sft_loss": 1.660813570022583,
      "step": 3540
    },
    {
      "epoch": 1.8973072420137147,
      "grad_norm": 7.6979737502482655,
      "learning_rate": 3.5792513196482373e-07,
      "logits/chosen": -0.3521515727043152,
      "logits/rejected": -0.04693521186709404,
      "logps/chosen": -1.546839714050293,
      "logps/rejected": -2.0670905113220215,
      "loss": 0.4612,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.546839714050293,
      "rewards/margins": 0.520250678062439,
      "rewards/rejected": -2.0670905113220215,
      "sft_loss": 1.6589651107788086,
      "step": 3545
    },
    {
      "epoch": 1.8999832747951162,
      "grad_norm": 7.046326412078959,
      "learning_rate": 3.5643261718944346e-07,
      "logits/chosen": -0.12103285640478134,
      "logits/rejected": -0.01766505278646946,
      "logps/chosen": -1.55022394657135,
      "logps/rejected": -1.9817107915878296,
      "loss": 0.5041,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.55022394657135,
      "rewards/margins": 0.4314870834350586,
      "rewards/rejected": -1.9817107915878296,
      "sft_loss": 1.5364574193954468,
      "step": 3550
    },
    {
      "epoch": 1.902659307576518,
      "grad_norm": 7.880188076428199,
      "learning_rate": 3.5494149530544087e-07,
      "logits/chosen": -0.27796703577041626,
      "logits/rejected": -0.11648659408092499,
      "logps/chosen": -1.479952096939087,
      "logps/rejected": -2.0038201808929443,
      "loss": 0.49,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.479952096939087,
      "rewards/margins": 0.5238681435585022,
      "rewards/rejected": -2.0038201808929443,
      "sft_loss": 1.5604270696640015,
      "step": 3555
    },
    {
      "epoch": 1.9053353403579194,
      "grad_norm": 7.573845511505143,
      "learning_rate": 3.534517807796871e-07,
      "logits/chosen": -0.16289106011390686,
      "logits/rejected": -0.06578390300273895,
      "logps/chosen": -1.5211942195892334,
      "logps/rejected": -2.022411584854126,
      "loss": 0.4725,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5211942195892334,
      "rewards/margins": 0.5012173056602478,
      "rewards/rejected": -2.022411584854126,
      "sft_loss": 1.5894739627838135,
      "step": 3560
    },
    {
      "epoch": 1.908011373139321,
      "grad_norm": 5.366824693587982,
      "learning_rate": 3.519634880653988e-07,
      "logits/chosen": -0.16283658146858215,
      "logits/rejected": -0.07833613455295563,
      "logps/chosen": -1.460172414779663,
      "logps/rejected": -2.1396899223327637,
      "loss": 0.4387,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.460172414779663,
      "rewards/margins": 0.6795173287391663,
      "rewards/rejected": -2.1396899223327637,
      "sft_loss": 1.607731819152832,
      "step": 3565
    },
    {
      "epoch": 1.9106874059207226,
      "grad_norm": 7.529517971883579,
      "learning_rate": 3.504766316019987e-07,
      "logits/chosen": -0.23355989158153534,
      "logits/rejected": -0.07068635523319244,
      "logps/chosen": -1.5029217004776,
      "logps/rejected": -2.018176794052124,
      "loss": 0.4548,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5029217004776,
      "rewards/margins": 0.5152554512023926,
      "rewards/rejected": -2.018176794052124,
      "sft_loss": 1.5598933696746826,
      "step": 3570
    },
    {
      "epoch": 1.913363438702124,
      "grad_norm": 5.399445925758498,
      "learning_rate": 3.489912258149745e-07,
      "logits/chosen": -0.10982157289981842,
      "logits/rejected": 0.03126415237784386,
      "logps/chosen": -1.5024158954620361,
      "logps/rejected": -2.045030355453491,
      "loss": 0.4849,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5024158954620361,
      "rewards/margins": 0.5426144003868103,
      "rewards/rejected": -2.045030355453491,
      "sft_loss": 1.5932767391204834,
      "step": 3575
    },
    {
      "epoch": 1.9160394714835256,
      "grad_norm": 5.710608786017775,
      "learning_rate": 3.475072851157397e-07,
      "logits/chosen": -0.1709875911474228,
      "logits/rejected": -0.10945823043584824,
      "logps/chosen": -1.4984357357025146,
      "logps/rejected": -2.104400634765625,
      "loss": 0.4279,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4984357357025146,
      "rewards/margins": 0.6059646606445312,
      "rewards/rejected": -2.104400634765625,
      "sft_loss": 1.5858982801437378,
      "step": 3580
    },
    {
      "epoch": 1.9187155042649273,
      "grad_norm": 7.429716993450721,
      "learning_rate": 3.460248239014936e-07,
      "logits/chosen": -0.08576101809740067,
      "logits/rejected": -0.017979338765144348,
      "logps/chosen": -1.6544307470321655,
      "logps/rejected": -2.1500606536865234,
      "loss": 0.4825,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6544307470321655,
      "rewards/margins": 0.4956297278404236,
      "rewards/rejected": -2.1500606536865234,
      "sft_loss": 1.7742359638214111,
      "step": 3585
    },
    {
      "epoch": 1.9213915370463288,
      "grad_norm": 7.303360436549522,
      "learning_rate": 3.4454385655508134e-07,
      "logits/chosen": -0.13903836905956268,
      "logits/rejected": -0.05833768844604492,
      "logps/chosen": -1.598507285118103,
      "logps/rejected": -1.9958927631378174,
      "loss": 0.5269,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.598507285118103,
      "rewards/margins": 0.3973854184150696,
      "rewards/rejected": -1.9958927631378174,
      "sft_loss": 1.6810901165008545,
      "step": 3590
    },
    {
      "epoch": 1.9240675698277303,
      "grad_norm": 6.7050619799100195,
      "learning_rate": 3.4306439744485447e-07,
      "logits/chosen": -0.29747599363327026,
      "logits/rejected": -0.07863298803567886,
      "logps/chosen": -1.525634527206421,
      "logps/rejected": -2.047628879547119,
      "loss": 0.4734,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.525634527206421,
      "rewards/margins": 0.521994411945343,
      "rewards/rejected": -2.047628879547119,
      "sft_loss": 1.496787667274475,
      "step": 3595
    },
    {
      "epoch": 1.926743602609132,
      "grad_norm": 7.263561975702709,
      "learning_rate": 3.415864609245322e-07,
      "logits/chosen": -0.12187772989273071,
      "logits/rejected": 0.06612186878919601,
      "logps/chosen": -1.561301827430725,
      "logps/rejected": -2.2543601989746094,
      "loss": 0.4712,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.561301827430725,
      "rewards/margins": 0.6930583715438843,
      "rewards/rejected": -2.2543601989746094,
      "sft_loss": 1.6595875024795532,
      "step": 3600
    },
    {
      "epoch": 1.926743602609132,
      "eval_logits/chosen": 0.20892609655857086,
      "eval_logits/rejected": 0.31129610538482666,
      "eval_logps/chosen": -1.564028024673462,
      "eval_logps/rejected": -2.076958179473877,
      "eval_loss": 0.4893736243247986,
      "eval_rewards/accuracies": 0.6661720871925354,
      "eval_rewards/chosen": -1.564028024673462,
      "eval_rewards/margins": 0.5129303336143494,
      "eval_rewards/rejected": -2.076958179473877,
      "eval_runtime": 43.2822,
      "eval_samples_per_second": 31.075,
      "eval_sft_loss": 1.6467366218566895,
      "eval_steps_per_second": 7.786,
      "step": 3600
    },
    {
      "epoch": 1.9294196353905335,
      "grad_norm": 7.70326214265193,
      "learning_rate": 3.401100613330605e-07,
      "logits/chosen": -0.21333487331867218,
      "logits/rejected": -0.16703540086746216,
      "logps/chosen": -1.5174494981765747,
      "logps/rejected": -1.9064096212387085,
      "loss": 0.5063,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5174494981765747,
      "rewards/margins": 0.3889603018760681,
      "rewards/rejected": -1.9064096212387085,
      "sft_loss": 1.6115996837615967,
      "step": 3605
    },
    {
      "epoch": 1.932095668171935,
      "grad_norm": 6.484089558922941,
      "learning_rate": 3.3863521299447514e-07,
      "logits/chosen": -0.17045536637306213,
      "logits/rejected": -0.04207264631986618,
      "logps/chosen": -1.5239019393920898,
      "logps/rejected": -2.0207948684692383,
      "loss": 0.4741,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5239019393920898,
      "rewards/margins": 0.49689292907714844,
      "rewards/rejected": -2.0207948684692383,
      "sft_loss": 1.6819967031478882,
      "step": 3610
    },
    {
      "epoch": 1.9347717009533367,
      "grad_norm": 6.901103706125059,
      "learning_rate": 3.371619302177609e-07,
      "logits/chosen": -0.11860332638025284,
      "logits/rejected": 0.0086726825684309,
      "logps/chosen": -1.583103895187378,
      "logps/rejected": -2.07374906539917,
      "loss": 0.4924,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.583103895187378,
      "rewards/margins": 0.4906453490257263,
      "rewards/rejected": -2.07374906539917,
      "sft_loss": 1.6645939350128174,
      "step": 3615
    },
    {
      "epoch": 1.9374477337347382,
      "grad_norm": 11.67293328945311,
      "learning_rate": 3.3569022729671393e-07,
      "logits/chosen": -0.16560427844524384,
      "logits/rejected": -0.08525630831718445,
      "logps/chosen": -1.642525315284729,
      "logps/rejected": -2.016528844833374,
      "loss": 0.5224,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.642525315284729,
      "rewards/margins": 0.3740035593509674,
      "rewards/rejected": -2.016528844833374,
      "sft_loss": 1.735414743423462,
      "step": 3620
    },
    {
      "epoch": 1.9401237665161397,
      "grad_norm": 7.779367874057267,
      "learning_rate": 3.342201185098024e-07,
      "logits/chosen": -0.10910829156637192,
      "logits/rejected": -0.07743573188781738,
      "logps/chosen": -1.5290305614471436,
      "logps/rejected": -1.9644886255264282,
      "loss": 0.4928,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5290305614471436,
      "rewards/margins": 0.43545812368392944,
      "rewards/rejected": -1.9644886255264282,
      "sft_loss": 1.6208540201187134,
      "step": 3625
    },
    {
      "epoch": 1.9427997992975414,
      "grad_norm": 13.173882267171926,
      "learning_rate": 3.3275161812002807e-07,
      "logits/chosen": -0.20582859218120575,
      "logits/rejected": -0.1449136734008789,
      "logps/chosen": -1.5608258247375488,
      "logps/rejected": -1.9999399185180664,
      "loss": 0.5364,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.5608258247375488,
      "rewards/margins": 0.43911415338516235,
      "rewards/rejected": -1.9999399185180664,
      "sft_loss": 1.7072786092758179,
      "step": 3630
    },
    {
      "epoch": 1.945475832078943,
      "grad_norm": 7.197716852516967,
      "learning_rate": 3.312847403747883e-07,
      "logits/chosen": -0.22931042313575745,
      "logits/rejected": -0.13184943795204163,
      "logps/chosen": -1.4989603757858276,
      "logps/rejected": -1.9964017868041992,
      "loss": 0.4713,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4989603757858276,
      "rewards/margins": 0.49744129180908203,
      "rewards/rejected": -1.9964017868041992,
      "sft_loss": 1.6010043621063232,
      "step": 3635
    },
    {
      "epoch": 1.9481518648603444,
      "grad_norm": 6.03372519429463,
      "learning_rate": 3.2981949950573733e-07,
      "logits/chosen": -0.15877914428710938,
      "logits/rejected": -0.040464289486408234,
      "logps/chosen": -1.6278139352798462,
      "logps/rejected": -1.9819543361663818,
      "loss": 0.5118,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.6278139352798462,
      "rewards/margins": 0.3541404604911804,
      "rewards/rejected": -1.9819543361663818,
      "sft_loss": 1.699200987815857,
      "step": 3640
    },
    {
      "epoch": 1.9508278976417461,
      "grad_norm": 6.5355149361133895,
      "learning_rate": 3.283559097286486e-07,
      "logits/chosen": -0.20665928721427917,
      "logits/rejected": -0.06811292469501495,
      "logps/chosen": -1.6473109722137451,
      "logps/rejected": -1.948591947555542,
      "loss": 0.5372,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.6473109722137451,
      "rewards/margins": 0.3012810945510864,
      "rewards/rejected": -1.948591947555542,
      "sft_loss": 1.7403411865234375,
      "step": 3645
    },
    {
      "epoch": 1.9535039304231478,
      "grad_norm": 11.784323927993501,
      "learning_rate": 3.268939852432765e-07,
      "logits/chosen": -0.24116232991218567,
      "logits/rejected": -0.12919065356254578,
      "logps/chosen": -1.5015369653701782,
      "logps/rejected": -1.898349404335022,
      "loss": 0.4851,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5015369653701782,
      "rewards/margins": 0.3968126177787781,
      "rewards/rejected": -1.898349404335022,
      "sft_loss": 1.6438575983047485,
      "step": 3650
    },
    {
      "epoch": 1.9561799632045491,
      "grad_norm": 7.467011155078924,
      "learning_rate": 3.254337402332187e-07,
      "logits/chosen": -0.18975183367729187,
      "logits/rejected": -0.050173837691545486,
      "logps/chosen": -1.5932409763336182,
      "logps/rejected": -2.0303874015808105,
      "loss": 0.5,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5932409763336182,
      "rewards/margins": 0.4371464252471924,
      "rewards/rejected": -2.0303874015808105,
      "sft_loss": 1.6250934600830078,
      "step": 3655
    },
    {
      "epoch": 1.9588559959859508,
      "grad_norm": 10.23729026474453,
      "learning_rate": 3.239751888657788e-07,
      "logits/chosen": -0.24026088416576385,
      "logits/rejected": -0.1027134507894516,
      "logps/chosen": -1.456146478652954,
      "logps/rejected": -1.9496123790740967,
      "loss": 0.4637,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.456146478652954,
      "rewards/margins": 0.4934659004211426,
      "rewards/rejected": -1.9496123790740967,
      "sft_loss": 1.5960451364517212,
      "step": 3660
    },
    {
      "epoch": 1.9615320287673526,
      "grad_norm": 7.21246606914941,
      "learning_rate": 3.2251834529182856e-07,
      "logits/chosen": -0.16082945466041565,
      "logits/rejected": -0.03842328116297722,
      "logps/chosen": -1.398827075958252,
      "logps/rejected": -1.8935636281967163,
      "loss": 0.4908,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.398827075958252,
      "rewards/margins": 0.49473652243614197,
      "rewards/rejected": -1.8935636281967163,
      "sft_loss": 1.4228825569152832,
      "step": 3665
    },
    {
      "epoch": 1.9642080615487538,
      "grad_norm": 6.769604855335532,
      "learning_rate": 3.2106322364567075e-07,
      "logits/chosen": -0.22607287764549255,
      "logits/rejected": -0.06951910257339478,
      "logps/chosen": -1.4958351850509644,
      "logps/rejected": -2.083021640777588,
      "loss": 0.4514,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4958351850509644,
      "rewards/margins": 0.5871865153312683,
      "rewards/rejected": -2.083021640777588,
      "sft_loss": 1.6325403451919556,
      "step": 3670
    },
    {
      "epoch": 1.9668840943301555,
      "grad_norm": 7.027012410446907,
      "learning_rate": 3.1960983804490183e-07,
      "logits/chosen": -0.2434675395488739,
      "logits/rejected": -0.08809126913547516,
      "logps/chosen": -1.6265697479248047,
      "logps/rejected": -2.3062081336975098,
      "loss": 0.4797,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.6265697479248047,
      "rewards/margins": 0.6796378493309021,
      "rewards/rejected": -2.3062081336975098,
      "sft_loss": 1.6968345642089844,
      "step": 3675
    },
    {
      "epoch": 1.9695601271115573,
      "grad_norm": 10.161618680788564,
      "learning_rate": 3.1815820259027537e-07,
      "logits/chosen": -0.2174391746520996,
      "logits/rejected": -0.08748964965343475,
      "logps/chosen": -1.3686556816101074,
      "logps/rejected": -1.8687818050384521,
      "loss": 0.4293,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3686556816101074,
      "rewards/margins": 0.5001261234283447,
      "rewards/rejected": -1.8687818050384521,
      "sft_loss": 1.4801901578903198,
      "step": 3680
    },
    {
      "epoch": 1.9722361598929585,
      "grad_norm": 8.215662645393254,
      "learning_rate": 3.16708331365565e-07,
      "logits/chosen": -0.2564537525177002,
      "logits/rejected": -0.16647759079933167,
      "logps/chosen": -1.4997940063476562,
      "logps/rejected": -2.045769214630127,
      "loss": 0.4809,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4997940063476562,
      "rewards/margins": 0.5459750890731812,
      "rewards/rejected": -2.045769214630127,
      "sft_loss": 1.6288654804229736,
      "step": 3685
    },
    {
      "epoch": 1.9749121926743602,
      "grad_norm": 6.30101921158305,
      "learning_rate": 3.152602384374275e-07,
      "logits/chosen": -0.20720060169696808,
      "logits/rejected": -0.036762163043022156,
      "logps/chosen": -1.5758286714553833,
      "logps/rejected": -2.0885355472564697,
      "loss": 0.487,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5758286714553833,
      "rewards/margins": 0.5127067565917969,
      "rewards/rejected": -2.0885355472564697,
      "sft_loss": 1.5761165618896484,
      "step": 3690
    },
    {
      "epoch": 1.977588225455762,
      "grad_norm": 7.226297337613594,
      "learning_rate": 3.1381393785526697e-07,
      "logits/chosen": -0.15174725651741028,
      "logits/rejected": -0.09643477201461792,
      "logps/chosen": -1.5932036638259888,
      "logps/rejected": -2.095515727996826,
      "loss": 0.5091,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5932036638259888,
      "rewards/margins": 0.5023123025894165,
      "rewards/rejected": -2.095515727996826,
      "sft_loss": 1.7102028131484985,
      "step": 3695
    },
    {
      "epoch": 1.9802642582371635,
      "grad_norm": 6.184587597685185,
      "learning_rate": 3.123694436510979e-07,
      "logits/chosen": -0.12151547521352768,
      "logits/rejected": 0.0030573017429560423,
      "logps/chosen": -1.4618427753448486,
      "logps/rejected": -1.9577305316925049,
      "loss": 0.4717,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4618427753448486,
      "rewards/margins": 0.4958876669406891,
      "rewards/rejected": -1.9577305316925049,
      "sft_loss": 1.5311295986175537,
      "step": 3700
    },
    {
      "epoch": 1.982940291018565,
      "grad_norm": 6.31477109275785,
      "learning_rate": 3.1092676983940946e-07,
      "logits/chosen": -0.18775346875190735,
      "logits/rejected": -0.10839760303497314,
      "logps/chosen": -1.5057272911071777,
      "logps/rejected": -1.9904956817626953,
      "loss": 0.4825,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5057272911071777,
      "rewards/margins": 0.4847683906555176,
      "rewards/rejected": -1.9904956817626953,
      "sft_loss": 1.5433039665222168,
      "step": 3705
    },
    {
      "epoch": 1.9856163237999667,
      "grad_norm": 8.486572731436723,
      "learning_rate": 3.094859304170293e-07,
      "logits/chosen": -0.04733508080244064,
      "logits/rejected": 0.025062087923288345,
      "logps/chosen": -1.5226056575775146,
      "logps/rejected": -1.9573904275894165,
      "loss": 0.5171,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5226056575775146,
      "rewards/margins": 0.43478474020957947,
      "rewards/rejected": -1.9573904275894165,
      "sft_loss": 1.640038251876831,
      "step": 3710
    },
    {
      "epoch": 1.9882923565813682,
      "grad_norm": 5.509472701150456,
      "learning_rate": 3.0804693936298795e-07,
      "logits/chosen": -0.14096835255622864,
      "logits/rejected": -0.06170610338449478,
      "logps/chosen": -1.582094430923462,
      "logps/rejected": -2.0880839824676514,
      "loss": 0.4967,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.582094430923462,
      "rewards/margins": 0.5059898495674133,
      "rewards/rejected": -2.0880839824676514,
      "sft_loss": 1.6778017282485962,
      "step": 3715
    },
    {
      "epoch": 1.9909683893627697,
      "grad_norm": 6.147956944803346,
      "learning_rate": 3.066098106383826e-07,
      "logits/chosen": -0.18321634829044342,
      "logits/rejected": -0.09819085896015167,
      "logps/chosen": -1.4959570169448853,
      "logps/rejected": -1.8988326787948608,
      "loss": 0.4881,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4959570169448853,
      "rewards/margins": 0.40287572145462036,
      "rewards/rejected": -1.8988326787948608,
      "sft_loss": 1.531747817993164,
      "step": 3720
    },
    {
      "epoch": 1.9936444221441714,
      "grad_norm": 5.382806009815984,
      "learning_rate": 3.0517455818624263e-07,
      "logits/chosen": -0.26531511545181274,
      "logits/rejected": -0.15976980328559875,
      "logps/chosen": -1.523664951324463,
      "logps/rejected": -1.9966137409210205,
      "loss": 0.5068,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.523664951324463,
      "rewards/margins": 0.47294870018959045,
      "rewards/rejected": -1.9966137409210205,
      "sft_loss": 1.676461935043335,
      "step": 3725
    },
    {
      "epoch": 1.9963204549255729,
      "grad_norm": 8.441690388912436,
      "learning_rate": 3.037411959313936e-07,
      "logits/chosen": -0.15571698546409607,
      "logits/rejected": -0.014239877462387085,
      "logps/chosen": -1.4310564994812012,
      "logps/rejected": -1.9260032176971436,
      "loss": 0.4485,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4310564994812012,
      "rewards/margins": 0.4949464797973633,
      "rewards/rejected": -1.9260032176971436,
      "sft_loss": 1.4834562540054321,
      "step": 3730
    },
    {
      "epoch": 1.9989964877069744,
      "grad_norm": 9.974130353603462,
      "learning_rate": 3.023097377803224e-07,
      "logits/chosen": -0.1044548749923706,
      "logits/rejected": -0.014732305891811848,
      "logps/chosen": -1.598559856414795,
      "logps/rejected": -1.9291934967041016,
      "loss": 0.5873,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.598559856414795,
      "rewards/margins": 0.3306335508823395,
      "rewards/rejected": -1.9291934967041016,
      "sft_loss": 1.6485782861709595,
      "step": 3735
    },
    {
      "epoch": 2.001672520488376,
      "grad_norm": 7.78305620035311,
      "learning_rate": 3.008801976210423e-07,
      "logits/chosen": -0.12677548825740814,
      "logits/rejected": -0.05798447132110596,
      "logps/chosen": -1.6210359334945679,
      "logps/rejected": -1.9703184366226196,
      "loss": 0.5402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6210359334945679,
      "rewards/margins": 0.34928256273269653,
      "rewards/rejected": -1.9703184366226196,
      "sft_loss": 1.632920265197754,
      "step": 3740
    },
    {
      "epoch": 2.0043485532697773,
      "grad_norm": 8.74331278022788,
      "learning_rate": 2.994525893229581e-07,
      "logits/chosen": -0.1670970916748047,
      "logits/rejected": -0.06462232768535614,
      "logps/chosen": -1.592901349067688,
      "logps/rejected": -2.0148890018463135,
      "loss": 0.5052,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.592901349067688,
      "rewards/margins": 0.42198771238327026,
      "rewards/rejected": -2.0148890018463135,
      "sft_loss": 1.6603543758392334,
      "step": 3745
    },
    {
      "epoch": 2.007024586051179,
      "grad_norm": 7.576250940089037,
      "learning_rate": 2.98026926736732e-07,
      "logits/chosen": -0.2261175662279129,
      "logits/rejected": -0.12932650744915009,
      "logps/chosen": -1.403714895248413,
      "logps/rejected": -2.009172201156616,
      "loss": 0.4393,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.403714895248413,
      "rewards/margins": 0.6054573655128479,
      "rewards/rejected": -2.009172201156616,
      "sft_loss": 1.5091825723648071,
      "step": 3750
    },
    {
      "epoch": 2.0097006188325808,
      "grad_norm": 7.69555538626666,
      "learning_rate": 2.9660322369414846e-07,
      "logits/chosen": -0.19793611764907837,
      "logits/rejected": -0.09182445704936981,
      "logps/chosen": -1.4750864505767822,
      "logps/rejected": -2.129772901535034,
      "loss": 0.4299,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4750864505767822,
      "rewards/margins": 0.654686450958252,
      "rewards/rejected": -2.129772901535034,
      "sft_loss": 1.6487104892730713,
      "step": 3755
    },
    {
      "epoch": 2.0123766516139825,
      "grad_norm": 5.695043205810022,
      "learning_rate": 2.9518149400798063e-07,
      "logits/chosen": -0.2694675624370575,
      "logits/rejected": -0.22962899506092072,
      "logps/chosen": -1.5076862573623657,
      "logps/rejected": -2.1112308502197266,
      "loss": 0.4655,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5076862573623657,
      "rewards/margins": 0.6035445928573608,
      "rewards/rejected": -2.1112308502197266,
      "sft_loss": 1.6350364685058594,
      "step": 3760
    },
    {
      "epoch": 2.0150526843953838,
      "grad_norm": 11.1683649183914,
      "learning_rate": 2.9376175147185633e-07,
      "logits/chosen": -0.15088486671447754,
      "logits/rejected": 0.04365579038858414,
      "logps/chosen": -1.5514955520629883,
      "logps/rejected": -2.126537799835205,
      "loss": 0.4801,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5514955520629883,
      "rewards/margins": 0.5750420689582825,
      "rewards/rejected": -2.126537799835205,
      "sft_loss": 1.5818036794662476,
      "step": 3765
    },
    {
      "epoch": 2.0177287171767855,
      "grad_norm": 14.540247275731168,
      "learning_rate": 2.9234400986012376e-07,
      "logits/chosen": -0.2786027491092682,
      "logits/rejected": -0.10113555192947388,
      "logps/chosen": -1.4965102672576904,
      "logps/rejected": -2.2697319984436035,
      "loss": 0.4207,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4965102672576904,
      "rewards/margins": 0.7732216119766235,
      "rewards/rejected": -2.2697319984436035,
      "sft_loss": 1.6020082235336304,
      "step": 3770
    },
    {
      "epoch": 2.020404749958187,
      "grad_norm": 7.548258945982458,
      "learning_rate": 2.9092828292771817e-07,
      "logits/chosen": -0.22244305908679962,
      "logits/rejected": -0.15660223364830017,
      "logps/chosen": -1.5342885255813599,
      "logps/rejected": -2.0522589683532715,
      "loss": 0.4753,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.5342885255813599,
      "rewards/margins": 0.517970621585846,
      "rewards/rejected": -2.0522589683532715,
      "sft_loss": 1.6077263355255127,
      "step": 3775
    },
    {
      "epoch": 2.0230807827395885,
      "grad_norm": 6.56138098807187,
      "learning_rate": 2.8951458441002875e-07,
      "logits/chosen": -0.18377938866615295,
      "logits/rejected": -0.13642618060112,
      "logps/chosen": -1.5458452701568604,
      "logps/rejected": -2.121659994125366,
      "loss": 0.4729,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5458452701568604,
      "rewards/margins": 0.575814962387085,
      "rewards/rejected": -2.121659994125366,
      "sft_loss": 1.6719255447387695,
      "step": 3780
    },
    {
      "epoch": 2.02575681552099,
      "grad_norm": 4.89627946113857,
      "learning_rate": 2.881029280227643e-07,
      "logits/chosen": -0.21539512276649475,
      "logits/rejected": -0.07043374329805374,
      "logps/chosen": -1.6168758869171143,
      "logps/rejected": -2.2531332969665527,
      "loss": 0.4558,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.6168758869171143,
      "rewards/margins": 0.636257529258728,
      "rewards/rejected": -2.2531332969665527,
      "sft_loss": 1.6750949621200562,
      "step": 3785
    },
    {
      "epoch": 2.028432848302392,
      "grad_norm": 4.515232600959035,
      "learning_rate": 2.8669332746182177e-07,
      "logits/chosen": -0.26013562083244324,
      "logits/rejected": -0.07724117487668991,
      "logps/chosen": -1.5244057178497314,
      "logps/rejected": -2.172802686691284,
      "loss": 0.4265,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5244057178497314,
      "rewards/margins": 0.6483969688415527,
      "rewards/rejected": -2.172802686691284,
      "sft_loss": 1.6660228967666626,
      "step": 3790
    },
    {
      "epoch": 2.031108881083793,
      "grad_norm": 7.811442251227995,
      "learning_rate": 2.8528579640315156e-07,
      "logits/chosen": -0.16741736233234406,
      "logits/rejected": -0.13274990022182465,
      "logps/chosen": -1.447377324104309,
      "logps/rejected": -1.9069875478744507,
      "loss": 0.4801,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.447377324104309,
      "rewards/margins": 0.45961007475852966,
      "rewards/rejected": -1.9069875478744507,
      "sft_loss": 1.583846092224121,
      "step": 3795
    },
    {
      "epoch": 2.033784913865195,
      "grad_norm": 8.087433317762617,
      "learning_rate": 2.8388034850262646e-07,
      "logits/chosen": -0.173568457365036,
      "logits/rejected": -0.03578788787126541,
      "logps/chosen": -1.6015841960906982,
      "logps/rejected": -2.189645767211914,
      "loss": 0.4415,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6015841960906982,
      "rewards/margins": 0.5880613327026367,
      "rewards/rejected": -2.189645767211914,
      "sft_loss": 1.7482211589813232,
      "step": 3800
    },
    {
      "epoch": 2.0364609466465966,
      "grad_norm": 6.9505978866225115,
      "learning_rate": 2.824769973959079e-07,
      "logits/chosen": -0.1564304232597351,
      "logits/rejected": -0.0258103609085083,
      "logps/chosen": -1.4258654117584229,
      "logps/rejected": -1.9665100574493408,
      "loss": 0.4515,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4258654117584229,
      "rewards/margins": 0.540644645690918,
      "rewards/rejected": -1.9665100574493408,
      "sft_loss": 1.4974491596221924,
      "step": 3805
    },
    {
      "epoch": 2.039136979427998,
      "grad_norm": 7.860004423872975,
      "learning_rate": 2.81075756698315e-07,
      "logits/chosen": -0.0722806304693222,
      "logits/rejected": 0.023296961560845375,
      "logps/chosen": -1.4594520330429077,
      "logps/rejected": -2.125936269760132,
      "loss": 0.4245,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4594520330429077,
      "rewards/margins": 0.6664842963218689,
      "rewards/rejected": -2.125936269760132,
      "sft_loss": 1.5164482593536377,
      "step": 3810
    },
    {
      "epoch": 2.0418130122093996,
      "grad_norm": 8.150670426531857,
      "learning_rate": 2.7967664000469035e-07,
      "logits/chosen": -0.2723679840564728,
      "logits/rejected": -0.1410585194826126,
      "logps/chosen": -1.539659023284912,
      "logps/rejected": -2.0731496810913086,
      "loss": 0.4643,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.539659023284912,
      "rewards/margins": 0.5334906578063965,
      "rewards/rejected": -2.0731496810913086,
      "sft_loss": 1.551286220550537,
      "step": 3815
    },
    {
      "epoch": 2.0444890449908013,
      "grad_norm": 8.78976681786829,
      "learning_rate": 2.7827966088927095e-07,
      "logits/chosen": -0.2862371802330017,
      "logits/rejected": -0.061904869973659515,
      "logps/chosen": -1.6095263957977295,
      "logps/rejected": -2.1704697608947754,
      "loss": 0.4933,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6095263957977295,
      "rewards/margins": 0.5609431266784668,
      "rewards/rejected": -2.1704697608947754,
      "sft_loss": 1.6857903003692627,
      "step": 3820
    },
    {
      "epoch": 2.0471650777722026,
      "grad_norm": 8.155201143271022,
      "learning_rate": 2.768848329055538e-07,
      "logits/chosen": -0.21058204770088196,
      "logits/rejected": -0.09403979778289795,
      "logps/chosen": -1.4634374380111694,
      "logps/rejected": -2.047750949859619,
      "loss": 0.4233,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4634374380111694,
      "rewards/margins": 0.5843135118484497,
      "rewards/rejected": -2.047750949859619,
      "sft_loss": 1.5597999095916748,
      "step": 3825
    },
    {
      "epoch": 2.0498411105536043,
      "grad_norm": 6.740274381093628,
      "learning_rate": 2.7549216958616657e-07,
      "logits/chosen": -0.28222018480300903,
      "logits/rejected": -0.12227698415517807,
      "logps/chosen": -1.5999635457992554,
      "logps/rejected": -2.318425178527832,
      "loss": 0.4205,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5999635457992554,
      "rewards/margins": 0.7184616923332214,
      "rewards/rejected": -2.318425178527832,
      "sft_loss": 1.649235486984253,
      "step": 3830
    },
    {
      "epoch": 2.052517143335006,
      "grad_norm": 7.080030593128097,
      "learning_rate": 2.741016844427344e-07,
      "logits/chosen": -0.19885589182376862,
      "logits/rejected": -0.020628320053219795,
      "logps/chosen": -1.5972665548324585,
      "logps/rejected": -2.2121667861938477,
      "loss": 0.4569,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5972665548324585,
      "rewards/margins": 0.6149001717567444,
      "rewards/rejected": -2.2121667861938477,
      "sft_loss": 1.6844546794891357,
      "step": 3835
    },
    {
      "epoch": 2.0551931761164073,
      "grad_norm": 5.652564618824891,
      "learning_rate": 2.7271339096575073e-07,
      "logits/chosen": -0.1308698207139969,
      "logits/rejected": -0.0033002153504639864,
      "logps/chosen": -1.4605486392974854,
      "logps/rejected": -2.12688946723938,
      "loss": 0.428,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.4605486392974854,
      "rewards/margins": 0.6663408279418945,
      "rewards/rejected": -2.12688946723938,
      "sft_loss": 1.5540508031845093,
      "step": 3840
    },
    {
      "epoch": 2.057869208897809,
      "grad_norm": 4.777212634284139,
      "learning_rate": 2.713273026244446e-07,
      "logits/chosen": -0.33631742000579834,
      "logits/rejected": -0.0764494314789772,
      "logps/chosen": -1.634876012802124,
      "logps/rejected": -2.2439773082733154,
      "loss": 0.4502,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.634876012802124,
      "rewards/margins": 0.609101414680481,
      "rewards/rejected": -2.2439773082733154,
      "sft_loss": 1.6806738376617432,
      "step": 3845
    },
    {
      "epoch": 2.0605452416792107,
      "grad_norm": 6.863100762156229,
      "learning_rate": 2.6994343286665156e-07,
      "logits/chosen": -0.2605910897254944,
      "logits/rejected": -0.08244252949953079,
      "logps/chosen": -1.6415363550186157,
      "logps/rejected": -2.2351536750793457,
      "loss": 0.474,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.6415363550186157,
      "rewards/margins": 0.5936172604560852,
      "rewards/rejected": -2.2351536750793457,
      "sft_loss": 1.7238140106201172,
      "step": 3850
    },
    {
      "epoch": 2.063221274460612,
      "grad_norm": 13.936388029108453,
      "learning_rate": 2.6856179511868156e-07,
      "logits/chosen": -0.18211022019386292,
      "logits/rejected": 0.006972149014472961,
      "logps/chosen": -1.6002590656280518,
      "logps/rejected": -2.4544899463653564,
      "loss": 0.3998,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.6002590656280518,
      "rewards/margins": 0.8542307615280151,
      "rewards/rejected": -2.4544899463653564,
      "sft_loss": 1.63395094871521,
      "step": 3855
    },
    {
      "epoch": 2.0658973072420137,
      "grad_norm": 9.576481319148407,
      "learning_rate": 2.6718240278519056e-07,
      "logits/chosen": -0.214491605758667,
      "logits/rejected": -0.056644368916749954,
      "logps/chosen": -1.5494184494018555,
      "logps/rejected": -2.272798538208008,
      "loss": 0.4357,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5494184494018555,
      "rewards/margins": 0.7233801484107971,
      "rewards/rejected": -2.272798538208008,
      "sft_loss": 1.5816377401351929,
      "step": 3860
    },
    {
      "epoch": 2.0685733400234154,
      "grad_norm": 10.928117257713213,
      "learning_rate": 2.6580526924904866e-07,
      "logits/chosen": -0.31673842668533325,
      "logits/rejected": -0.11347083747386932,
      "logps/chosen": -1.6189310550689697,
      "logps/rejected": -2.098507881164551,
      "loss": 0.498,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6189310550689697,
      "rewards/margins": 0.4795767664909363,
      "rewards/rejected": -2.098507881164551,
      "sft_loss": 1.7159723043441772,
      "step": 3865
    },
    {
      "epoch": 2.0712493728048167,
      "grad_norm": 6.5000832860610025,
      "learning_rate": 2.6443040787121186e-07,
      "logits/chosen": -0.2351851910352707,
      "logits/rejected": -0.16655540466308594,
      "logps/chosen": -1.4487316608428955,
      "logps/rejected": -2.002185583114624,
      "loss": 0.4429,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4487316608428955,
      "rewards/margins": 0.5534540414810181,
      "rewards/rejected": -2.002185583114624,
      "sft_loss": 1.5564430952072144,
      "step": 3870
    },
    {
      "epoch": 2.0739254055862184,
      "grad_norm": 7.302439144857689,
      "learning_rate": 2.6305783199059084e-07,
      "logits/chosen": -0.2277861088514328,
      "logits/rejected": -0.11319579929113388,
      "logps/chosen": -1.6114695072174072,
      "logps/rejected": -2.214570999145508,
      "loss": 0.4271,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.6114695072174072,
      "rewards/margins": 0.6031014323234558,
      "rewards/rejected": -2.214570999145508,
      "sft_loss": 1.7454404830932617,
      "step": 3875
    },
    {
      "epoch": 2.07660143836762,
      "grad_norm": 11.473067531134857,
      "learning_rate": 2.6168755492392324e-07,
      "logits/chosen": -0.21884004771709442,
      "logits/rejected": -0.036847881972789764,
      "logps/chosen": -1.3591184616088867,
      "logps/rejected": -2.0386531352996826,
      "loss": 0.402,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3591184616088867,
      "rewards/margins": 0.6795347332954407,
      "rewards/rejected": -2.0386531352996826,
      "sft_loss": 1.4358450174331665,
      "step": 3880
    },
    {
      "epoch": 2.0792774711490214,
      "grad_norm": 7.667532299861024,
      "learning_rate": 2.6031958996564274e-07,
      "logits/chosen": -0.26414209604263306,
      "logits/rejected": -0.100743368268013,
      "logps/chosen": -1.4504120349884033,
      "logps/rejected": -2.148660898208618,
      "loss": 0.4313,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4504120349884033,
      "rewards/margins": 0.698248565196991,
      "rewards/rejected": -2.148660898208618,
      "sft_loss": 1.566084623336792,
      "step": 3885
    },
    {
      "epoch": 2.081953503930423,
      "grad_norm": 6.60739297994297,
      "learning_rate": 2.589539503877518e-07,
      "logits/chosen": -0.12682762742042542,
      "logits/rejected": -0.027794640511274338,
      "logps/chosen": -1.5632338523864746,
      "logps/rejected": -2.1642110347747803,
      "loss": 0.4926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5632338523864746,
      "rewards/margins": 0.6009770631790161,
      "rewards/rejected": -2.1642110347747803,
      "sft_loss": 1.6938755512237549,
      "step": 3890
    },
    {
      "epoch": 2.084629536711825,
      "grad_norm": 6.770016103035415,
      "learning_rate": 2.5759064943969125e-07,
      "logits/chosen": -0.2298419028520584,
      "logits/rejected": 0.02210015431046486,
      "logps/chosen": -1.5325865745544434,
      "logps/rejected": -2.1005892753601074,
      "loss": 0.475,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5325865745544434,
      "rewards/margins": 0.568002462387085,
      "rewards/rejected": -2.1005892753601074,
      "sft_loss": 1.596083641052246,
      "step": 3895
    },
    {
      "epoch": 2.087305569493226,
      "grad_norm": 12.56192636846323,
      "learning_rate": 2.562297003482131e-07,
      "logits/chosen": -0.10446188598871231,
      "logits/rejected": -0.08038032799959183,
      "logps/chosen": -1.5202323198318481,
      "logps/rejected": -2.0459377765655518,
      "loss": 0.4598,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5202323198318481,
      "rewards/margins": 0.5257053375244141,
      "rewards/rejected": -2.0459377765655518,
      "sft_loss": 1.617698311805725,
      "step": 3900
    },
    {
      "epoch": 2.089981602274628,
      "grad_norm": 6.756218665882484,
      "learning_rate": 2.548711163172512e-07,
      "logits/chosen": -0.17761339247226715,
      "logits/rejected": -0.07439003139734268,
      "logps/chosen": -1.5598061084747314,
      "logps/rejected": -2.0722603797912598,
      "loss": 0.4954,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5598061084747314,
      "rewards/margins": 0.5124542117118835,
      "rewards/rejected": -2.0722603797912598,
      "sft_loss": 1.5847612619400024,
      "step": 3905
    },
    {
      "epoch": 2.0926576350560295,
      "grad_norm": 6.298927224311675,
      "learning_rate": 2.53514910527794e-07,
      "logits/chosen": -0.13386814296245575,
      "logits/rejected": 0.00870521366596222,
      "logps/chosen": -1.3839236497879028,
      "logps/rejected": -1.9484390020370483,
      "loss": 0.4291,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.3839236497879028,
      "rewards/margins": 0.5645155310630798,
      "rewards/rejected": -1.9484390020370483,
      "sft_loss": 1.4806008338928223,
      "step": 3910
    },
    {
      "epoch": 2.095333667837431,
      "grad_norm": 8.760310055144243,
      "learning_rate": 2.5216109613775573e-07,
      "logits/chosen": -0.2275843620300293,
      "logits/rejected": -0.043379418551921844,
      "logps/chosen": -1.5875427722930908,
      "logps/rejected": -2.2296032905578613,
      "loss": 0.4628,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5875427722930908,
      "rewards/margins": 0.6420606970787048,
      "rewards/rejected": -2.2296032905578613,
      "sft_loss": 1.6525790691375732,
      "step": 3915
    },
    {
      "epoch": 2.0980097006188325,
      "grad_norm": 9.378349348436625,
      "learning_rate": 2.5080968628184993e-07,
      "logits/chosen": -0.21023735404014587,
      "logits/rejected": -0.03672776371240616,
      "logps/chosen": -1.5500128269195557,
      "logps/rejected": -2.308952808380127,
      "loss": 0.4324,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.5500128269195557,
      "rewards/margins": 0.7589399814605713,
      "rewards/rejected": -2.308952808380127,
      "sft_loss": 1.6003620624542236,
      "step": 3920
    },
    {
      "epoch": 2.1006857334002342,
      "grad_norm": 6.685879334267199,
      "learning_rate": 2.494606940714605e-07,
      "logits/chosen": -0.20083554089069366,
      "logits/rejected": -0.08228044211864471,
      "logps/chosen": -1.4767929315567017,
      "logps/rejected": -2.145826816558838,
      "loss": 0.4338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4767929315567017,
      "rewards/margins": 0.6690336465835571,
      "rewards/rejected": -2.145826816558838,
      "sft_loss": 1.572529911994934,
      "step": 3925
    },
    {
      "epoch": 2.103361766181636,
      "grad_norm": 5.737150130242227,
      "learning_rate": 2.4811413259451625e-07,
      "logits/chosen": -0.290099561214447,
      "logits/rejected": -0.12537804245948792,
      "logps/chosen": -1.524739384651184,
      "logps/rejected": -2.1593596935272217,
      "loss": 0.4434,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.524739384651184,
      "rewards/margins": 0.6346203684806824,
      "rewards/rejected": -2.1593596935272217,
      "sft_loss": 1.5483014583587646,
      "step": 3930
    },
    {
      "epoch": 2.106037798963037,
      "grad_norm": 6.2794896545930765,
      "learning_rate": 2.46770014915362e-07,
      "logits/chosen": -0.17081448435783386,
      "logits/rejected": -0.10958276689052582,
      "logps/chosen": -1.560088872909546,
      "logps/rejected": -2.201547622680664,
      "loss": 0.4691,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.560088872909546,
      "rewards/margins": 0.6414587497711182,
      "rewards/rejected": -2.201547622680664,
      "sft_loss": 1.574739694595337,
      "step": 3935
    },
    {
      "epoch": 2.108713831744439,
      "grad_norm": 10.42380917965788,
      "learning_rate": 2.45428354074634e-07,
      "logits/chosen": -0.2000219076871872,
      "logits/rejected": -0.1316196471452713,
      "logps/chosen": -1.4703319072723389,
      "logps/rejected": -2.215935707092285,
      "loss": 0.4202,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4703319072723389,
      "rewards/margins": 0.7456039786338806,
      "rewards/rejected": -2.215935707092285,
      "sft_loss": 1.495848298072815,
      "step": 3940
    },
    {
      "epoch": 2.1113898645258407,
      "grad_norm": 10.759877860016946,
      "learning_rate": 2.4408916308913105e-07,
      "logits/chosen": -0.21619954705238342,
      "logits/rejected": -0.03465006873011589,
      "logps/chosen": -1.6587110757827759,
      "logps/rejected": -2.1734070777893066,
      "loss": 0.5023,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.6587110757827759,
      "rewards/margins": 0.5146957635879517,
      "rewards/rejected": -2.1734070777893066,
      "sft_loss": 1.7548824548721313,
      "step": 3945
    },
    {
      "epoch": 2.114065897307242,
      "grad_norm": 11.314688619454033,
      "learning_rate": 2.4275245495169025e-07,
      "logits/chosen": -0.12167392671108246,
      "logits/rejected": 0.04851360246539116,
      "logps/chosen": -1.4907797574996948,
      "logps/rejected": -2.0790886878967285,
      "loss": 0.4597,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4907797574996948,
      "rewards/margins": 0.5883088707923889,
      "rewards/rejected": -2.0790886878967285,
      "sft_loss": 1.560671091079712,
      "step": 3950
    },
    {
      "epoch": 2.1167419300886436,
      "grad_norm": 6.600920678232209,
      "learning_rate": 2.414182426310597e-07,
      "logits/chosen": -0.2696961462497711,
      "logits/rejected": -0.17066015303134918,
      "logps/chosen": -1.4763388633728027,
      "logps/rejected": -2.1660704612731934,
      "loss": 0.4481,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4763388633728027,
      "rewards/margins": 0.6897314786911011,
      "rewards/rejected": -2.1660704612731934,
      "sft_loss": 1.5858443975448608,
      "step": 3955
    },
    {
      "epoch": 2.1194179628700454,
      "grad_norm": 10.608197576928287,
      "learning_rate": 2.400865390717734e-07,
      "logits/chosen": -0.14795371890068054,
      "logits/rejected": -0.02199488878250122,
      "logps/chosen": -1.5266574621200562,
      "logps/rejected": -2.2974257469177246,
      "loss": 0.4362,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5266574621200562,
      "rewards/margins": 0.7707680463790894,
      "rewards/rejected": -2.2974257469177246,
      "sft_loss": 1.6296428442001343,
      "step": 3960
    },
    {
      "epoch": 2.1220939956514466,
      "grad_norm": 7.218085338433899,
      "learning_rate": 2.3875735719402475e-07,
      "logits/chosen": -0.14386795461177826,
      "logits/rejected": -0.024950990453362465,
      "logps/chosen": -1.5040401220321655,
      "logps/rejected": -2.217085361480713,
      "loss": 0.4303,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.5040401220321655,
      "rewards/margins": 0.7130452990531921,
      "rewards/rejected": -2.217085361480713,
      "sft_loss": 1.6567966938018799,
      "step": 3965
    },
    {
      "epoch": 2.1247700284328483,
      "grad_norm": 6.661608023455321,
      "learning_rate": 2.3743070989354258e-07,
      "logits/chosen": -0.17446637153625488,
      "logits/rejected": -0.06813861429691315,
      "logps/chosen": -1.454469919204712,
      "logps/rejected": -2.1430160999298096,
      "loss": 0.4389,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.454469919204712,
      "rewards/margins": 0.6885461807250977,
      "rewards/rejected": -2.1430160999298096,
      "sft_loss": 1.603070616722107,
      "step": 3970
    },
    {
      "epoch": 2.12744606121425,
      "grad_norm": 9.347508736323672,
      "learning_rate": 2.3610661004146454e-07,
      "logits/chosen": -0.1273220181465149,
      "logits/rejected": -0.019476449117064476,
      "logps/chosen": -1.3374217748641968,
      "logps/rejected": -1.8788425922393799,
      "loss": 0.4138,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3374217748641968,
      "rewards/margins": 0.5414208173751831,
      "rewards/rejected": -1.8788425922393799,
      "sft_loss": 1.3960545063018799,
      "step": 3975
    },
    {
      "epoch": 2.1301220939956513,
      "grad_norm": 6.201290406117352,
      "learning_rate": 2.3478507048421314e-07,
      "logits/chosen": -0.2582013010978699,
      "logits/rejected": -0.1725609004497528,
      "logps/chosen": -1.3875207901000977,
      "logps/rejected": -2.1059584617614746,
      "loss": 0.4121,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3875207901000977,
      "rewards/margins": 0.7184377908706665,
      "rewards/rejected": -2.1059584617614746,
      "sft_loss": 1.572930932044983,
      "step": 3980
    },
    {
      "epoch": 2.132798126777053,
      "grad_norm": 9.37812664861454,
      "learning_rate": 2.334661040433713e-07,
      "logits/chosen": -0.26035767793655396,
      "logits/rejected": -0.14250019192695618,
      "logps/chosen": -1.5116602182388306,
      "logps/rejected": -2.161254644393921,
      "loss": 0.4345,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5116602182388306,
      "rewards/margins": 0.6495946645736694,
      "rewards/rejected": -2.161254644393921,
      "sft_loss": 1.6295945644378662,
      "step": 3985
    },
    {
      "epoch": 2.1354741595584548,
      "grad_norm": 7.841514997852571,
      "learning_rate": 2.321497235155568e-07,
      "logits/chosen": -0.31633514165878296,
      "logits/rejected": -0.17246940732002258,
      "logps/chosen": -1.4053174257278442,
      "logps/rejected": -2.0792293548583984,
      "loss": 0.4131,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4053174257278442,
      "rewards/margins": 0.6739118695259094,
      "rewards/rejected": -2.0792293548583984,
      "sft_loss": 1.5280014276504517,
      "step": 3990
    },
    {
      "epoch": 2.138150192339856,
      "grad_norm": 13.6967942623219,
      "learning_rate": 2.3083594167229965e-07,
      "logits/chosen": -0.35641390085220337,
      "logits/rejected": -0.08034927397966385,
      "logps/chosen": -1.5294626951217651,
      "logps/rejected": -2.208108425140381,
      "loss": 0.4388,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5294626951217651,
      "rewards/margins": 0.6786457896232605,
      "rewards/rejected": -2.208108425140381,
      "sft_loss": 1.6018537282943726,
      "step": 3995
    },
    {
      "epoch": 2.1408262251212578,
      "grad_norm": 11.98212137009441,
      "learning_rate": 2.295247712599167e-07,
      "logits/chosen": -0.21660706400871277,
      "logits/rejected": -0.10180102288722992,
      "logps/chosen": -1.4509875774383545,
      "logps/rejected": -2.1792187690734863,
      "loss": 0.4297,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4509875774383545,
      "rewards/margins": 0.7282313108444214,
      "rewards/rejected": -2.1792187690734863,
      "sft_loss": 1.5786430835723877,
      "step": 4000
    },
    {
      "epoch": 2.1408262251212578,
      "eval_logits/chosen": 0.13109426200389862,
      "eval_logits/rejected": 0.23107898235321045,
      "eval_logps/chosen": -1.5826647281646729,
      "eval_logps/rejected": -2.126399278640747,
      "eval_loss": 0.4894164204597473,
      "eval_rewards/accuracies": 0.669881284236908,
      "eval_rewards/chosen": -1.5826647281646729,
      "eval_rewards/margins": 0.5437345504760742,
      "eval_rewards/rejected": -2.126399278640747,
      "eval_runtime": 43.266,
      "eval_samples_per_second": 31.087,
      "eval_sft_loss": 1.6623671054840088,
      "eval_steps_per_second": 7.789,
      "step": 4000
    },
    {
      "epoch": 2.1435022579026595,
      "grad_norm": 6.774209271920777,
      "learning_rate": 2.2821622499938948e-07,
      "logits/chosen": -0.25786370038986206,
      "logits/rejected": -0.03186394274234772,
      "logps/chosen": -1.7039791345596313,
      "logps/rejected": -2.3183369636535645,
      "loss": 0.4881,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.7039791345596313,
      "rewards/margins": 0.6143580079078674,
      "rewards/rejected": -2.3183369636535645,
      "sft_loss": 1.7434800863265991,
      "step": 4005
    },
    {
      "epoch": 2.1461782906840607,
      "grad_norm": 11.204427632919344,
      "learning_rate": 2.269103155862391e-07,
      "logits/chosen": -0.25920170545578003,
      "logits/rejected": -0.14853177964687347,
      "logps/chosen": -1.5233911275863647,
      "logps/rejected": -2.025580883026123,
      "loss": 0.4807,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5233911275863647,
      "rewards/margins": 0.5021899938583374,
      "rewards/rejected": -2.025580883026123,
      "sft_loss": 1.608466386795044,
      "step": 4010
    },
    {
      "epoch": 2.1488543234654625,
      "grad_norm": 6.277393465452174,
      "learning_rate": 2.2560705569040483e-07,
      "logits/chosen": -0.250881552696228,
      "logits/rejected": 0.021695155650377274,
      "logps/chosen": -1.5364277362823486,
      "logps/rejected": -2.1099002361297607,
      "loss": 0.4782,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5364277362823486,
      "rewards/margins": 0.5734724402427673,
      "rewards/rejected": -2.1099002361297607,
      "sft_loss": 1.6285278797149658,
      "step": 4015
    },
    {
      "epoch": 2.151530356246864,
      "grad_norm": 10.927604344319644,
      "learning_rate": 2.2430645795611963e-07,
      "logits/chosen": -0.35845470428466797,
      "logits/rejected": -0.19380858540534973,
      "logps/chosen": -1.6200428009033203,
      "logps/rejected": -2.2093708515167236,
      "loss": 0.4705,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6200428009033203,
      "rewards/margins": 0.5893279314041138,
      "rewards/rejected": -2.2093708515167236,
      "sft_loss": 1.7426955699920654,
      "step": 4020
    },
    {
      "epoch": 2.1542063890282654,
      "grad_norm": 7.323032772336199,
      "learning_rate": 2.230085350017884e-07,
      "logits/chosen": -0.22688297927379608,
      "logits/rejected": -0.11204588413238525,
      "logps/chosen": -1.4713099002838135,
      "logps/rejected": -2.0694479942321777,
      "loss": 0.4489,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4713099002838135,
      "rewards/margins": 0.5981377363204956,
      "rewards/rejected": -2.0694479942321777,
      "sft_loss": 1.527937650680542,
      "step": 4025
    },
    {
      "epoch": 2.156882421809667,
      "grad_norm": 14.434754823938734,
      "learning_rate": 2.2171329941986554e-07,
      "logits/chosen": -0.2611876130104065,
      "logits/rejected": -0.16281744837760925,
      "logps/chosen": -1.4556105136871338,
      "logps/rejected": -2.102907657623291,
      "loss": 0.4324,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4556105136871338,
      "rewards/margins": 0.6472971439361572,
      "rewards/rejected": -2.102907657623291,
      "sft_loss": 1.5691444873809814,
      "step": 4030
    },
    {
      "epoch": 2.159558454591069,
      "grad_norm": 9.831723652563838,
      "learning_rate": 2.2042076377673202e-07,
      "logits/chosen": -0.24293653666973114,
      "logits/rejected": -0.19557669758796692,
      "logps/chosen": -1.4355570077896118,
      "logps/rejected": -1.8510463237762451,
      "loss": 0.4623,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4355570077896118,
      "rewards/margins": 0.4154892861843109,
      "rewards/rejected": -1.8510463237762451,
      "sft_loss": 1.569411277770996,
      "step": 4035
    },
    {
      "epoch": 2.16223448737247,
      "grad_norm": 7.351491599536689,
      "learning_rate": 2.1913094061257476e-07,
      "logits/chosen": -0.25733011960983276,
      "logits/rejected": -0.21016161143779755,
      "logps/chosen": -1.3895686864852905,
      "logps/rejected": -1.9499037265777588,
      "loss": 0.4579,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.3895686864852905,
      "rewards/margins": 0.5603350400924683,
      "rewards/rejected": -1.9499037265777588,
      "sft_loss": 1.4448275566101074,
      "step": 4040
    },
    {
      "epoch": 2.164910520153872,
      "grad_norm": 8.226102757499863,
      "learning_rate": 2.178438424412633e-07,
      "logits/chosen": -0.18892326951026917,
      "logits/rejected": -0.049190543591976166,
      "logps/chosen": -1.52090585231781,
      "logps/rejected": -2.069546699523926,
      "loss": 0.4744,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.52090585231781,
      "rewards/margins": 0.5486409664154053,
      "rewards/rejected": -2.069546699523926,
      "sft_loss": 1.6149247884750366,
      "step": 4045
    },
    {
      "epoch": 2.1675865529352736,
      "grad_norm": 7.705027333491286,
      "learning_rate": 2.165594817502302e-07,
      "logits/chosen": -0.2961641252040863,
      "logits/rejected": -0.18423901498317719,
      "logps/chosen": -1.5554035902023315,
      "logps/rejected": -2.0131006240844727,
      "loss": 0.4978,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5554035902023315,
      "rewards/margins": 0.4576972424983978,
      "rewards/rejected": -2.0131006240844727,
      "sft_loss": 1.6668485403060913,
      "step": 4050
    },
    {
      "epoch": 2.170262585716675,
      "grad_norm": 8.141312762737401,
      "learning_rate": 2.1527787100034806e-07,
      "logits/chosen": -0.18819668889045715,
      "logits/rejected": -0.13292866945266724,
      "logps/chosen": -1.5318689346313477,
      "logps/rejected": -1.9563630819320679,
      "loss": 0.4714,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5318689346313477,
      "rewards/margins": 0.42449426651000977,
      "rewards/rejected": -1.9563630819320679,
      "sft_loss": 1.5768083333969116,
      "step": 4055
    },
    {
      "epoch": 2.1729386184980766,
      "grad_norm": 10.039453100753366,
      "learning_rate": 2.1399902262581037e-07,
      "logits/chosen": -0.09409301728010178,
      "logits/rejected": 0.020703068003058434,
      "logps/chosen": -1.436989426612854,
      "logps/rejected": -1.986651062965393,
      "loss": 0.4476,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.436989426612854,
      "rewards/margins": 0.5496617555618286,
      "rewards/rejected": -1.986651062965393,
      "sft_loss": 1.5771474838256836,
      "step": 4060
    },
    {
      "epoch": 2.1756146512794783,
      "grad_norm": 8.027729882763833,
      "learning_rate": 2.127229490340094e-07,
      "logits/chosen": -0.31718340516090393,
      "logits/rejected": -0.21360819041728973,
      "logps/chosen": -1.5581392049789429,
      "logps/rejected": -2.234459400177002,
      "loss": 0.4471,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.5581392049789429,
      "rewards/margins": 0.6763203144073486,
      "rewards/rejected": -2.234459400177002,
      "sft_loss": 1.6486304998397827,
      "step": 4065
    },
    {
      "epoch": 2.1782906840608796,
      "grad_norm": 10.71593597455729,
      "learning_rate": 2.1144966260541698e-07,
      "logits/chosen": -0.20124073326587677,
      "logits/rejected": 0.020577391609549522,
      "logps/chosen": -1.5547935962677002,
      "logps/rejected": -2.3254096508026123,
      "loss": 0.4472,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5547935962677002,
      "rewards/margins": 0.7706161141395569,
      "rewards/rejected": -2.3254096508026123,
      "sft_loss": 1.7266238927841187,
      "step": 4070
    },
    {
      "epoch": 2.1809667168422813,
      "grad_norm": 9.236529459942533,
      "learning_rate": 2.1017917569346332e-07,
      "logits/chosen": -0.27028411626815796,
      "logits/rejected": -0.06531749665737152,
      "logps/chosen": -1.6120535135269165,
      "logps/rejected": -2.1328024864196777,
      "loss": 0.4762,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6120535135269165,
      "rewards/margins": 0.5207492113113403,
      "rewards/rejected": -2.1328024864196777,
      "sft_loss": 1.6699886322021484,
      "step": 4075
    },
    {
      "epoch": 2.183642749623683,
      "grad_norm": 7.3321692955545545,
      "learning_rate": 2.0891150062441837e-07,
      "logits/chosen": -0.243181511759758,
      "logits/rejected": -0.09978292882442474,
      "logps/chosen": -1.5708192586898804,
      "logps/rejected": -2.295379638671875,
      "loss": 0.4431,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5708192586898804,
      "rewards/margins": 0.7245603203773499,
      "rewards/rejected": -2.295379638671875,
      "sft_loss": 1.692696213722229,
      "step": 4080
    },
    {
      "epoch": 2.1863187824050843,
      "grad_norm": 9.164383434433807,
      "learning_rate": 2.0764664969727086e-07,
      "logits/chosen": -0.22102339565753937,
      "logits/rejected": -0.11209341138601303,
      "logps/chosen": -1.4797003269195557,
      "logps/rejected": -2.004822254180908,
      "loss": 0.4526,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4797003269195557,
      "rewards/margins": 0.5251215696334839,
      "rewards/rejected": -2.004822254180908,
      "sft_loss": 1.5114164352416992,
      "step": 4085
    },
    {
      "epoch": 2.188994815186486,
      "grad_norm": 9.347179709940297,
      "learning_rate": 2.0638463518361033e-07,
      "logits/chosen": -0.30683061480522156,
      "logits/rejected": -0.09695632755756378,
      "logps/chosen": -1.4971176385879517,
      "logps/rejected": -2.1254069805145264,
      "loss": 0.4462,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4971176385879517,
      "rewards/margins": 0.6282894611358643,
      "rewards/rejected": -2.1254069805145264,
      "sft_loss": 1.6113370656967163,
      "step": 4090
    },
    {
      "epoch": 2.1916708479678877,
      "grad_norm": 5.754956838679529,
      "learning_rate": 2.0512546932750702e-07,
      "logits/chosen": -0.2652248740196228,
      "logits/rejected": -0.1713368147611618,
      "logps/chosen": -1.588734745979309,
      "logps/rejected": -2.086512804031372,
      "loss": 0.4839,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.588734745979309,
      "rewards/margins": 0.4977780878543854,
      "rewards/rejected": -2.086512804031372,
      "sft_loss": 1.6886154413223267,
      "step": 4095
    },
    {
      "epoch": 2.194346880749289,
      "grad_norm": 7.582408807269065,
      "learning_rate": 2.0386916434539343e-07,
      "logits/chosen": -0.19961073994636536,
      "logits/rejected": -0.05144035071134567,
      "logps/chosen": -1.4022175073623657,
      "logps/rejected": -2.1321256160736084,
      "loss": 0.407,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4022175073623657,
      "rewards/margins": 0.7299081683158875,
      "rewards/rejected": -2.1321256160736084,
      "sft_loss": 1.5723204612731934,
      "step": 4100
    },
    {
      "epoch": 2.1970229135306907,
      "grad_norm": 7.244622856694702,
      "learning_rate": 2.0261573242594627e-07,
      "logits/chosen": -0.23503056168556213,
      "logits/rejected": -0.02701449953019619,
      "logps/chosen": -1.6697590351104736,
      "logps/rejected": -2.253340482711792,
      "loss": 0.482,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6697590351104736,
      "rewards/margins": 0.583581268787384,
      "rewards/rejected": -2.253340482711792,
      "sft_loss": 1.7181288003921509,
      "step": 4105
    },
    {
      "epoch": 2.1996989463120924,
      "grad_norm": 16.411061274639884,
      "learning_rate": 2.0136518572996724e-07,
      "logits/chosen": -0.20764736831188202,
      "logits/rejected": -0.016535207629203796,
      "logps/chosen": -1.5405995845794678,
      "logps/rejected": -2.188555955886841,
      "loss": 0.4545,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5405995845794678,
      "rewards/margins": 0.647956371307373,
      "rewards/rejected": -2.188555955886841,
      "sft_loss": 1.6755081415176392,
      "step": 4110
    },
    {
      "epoch": 2.202374979093494,
      "grad_norm": 10.865795852770049,
      "learning_rate": 2.0011753639026617e-07,
      "logits/chosen": -0.18737569451332092,
      "logits/rejected": -0.12781764566898346,
      "logps/chosen": -1.585011601448059,
      "logps/rejected": -2.272808074951172,
      "loss": 0.4306,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.585011601448059,
      "rewards/margins": 0.6877964735031128,
      "rewards/rejected": -2.272808074951172,
      "sft_loss": 1.6630890369415283,
      "step": 4115
    },
    {
      "epoch": 2.2050510118748954,
      "grad_norm": 8.067074691179583,
      "learning_rate": 1.988727965115421e-07,
      "logits/chosen": -0.22606685757637024,
      "logits/rejected": -0.1338723599910736,
      "logps/chosen": -1.4738590717315674,
      "logps/rejected": -2.151301622390747,
      "loss": 0.4325,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4738590717315674,
      "rewards/margins": 0.6774424314498901,
      "rewards/rejected": -2.151301622390747,
      "sft_loss": 1.6365344524383545,
      "step": 4120
    },
    {
      "epoch": 2.207727044656297,
      "grad_norm": 6.89501685383988,
      "learning_rate": 1.9763097817026713e-07,
      "logits/chosen": -0.2842291295528412,
      "logits/rejected": -0.04498632252216339,
      "logps/chosen": -1.5477343797683716,
      "logps/rejected": -2.343984842300415,
      "loss": 0.4111,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5477343797683716,
      "rewards/margins": 0.7962502241134644,
      "rewards/rejected": -2.343984842300415,
      "sft_loss": 1.683488130569458,
      "step": 4125
    },
    {
      "epoch": 2.210403077437699,
      "grad_norm": 12.959262244969201,
      "learning_rate": 1.9639209341456796e-07,
      "logits/chosen": -0.19628119468688965,
      "logits/rejected": -0.10231286287307739,
      "logps/chosen": -1.5733191967010498,
      "logps/rejected": -2.239591360092163,
      "loss": 0.4725,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5733191967010498,
      "rewards/margins": 0.6662724614143372,
      "rewards/rejected": -2.239591360092163,
      "sft_loss": 1.6991240978240967,
      "step": 4130
    },
    {
      "epoch": 2.2130791102191,
      "grad_norm": 8.011736146801448,
      "learning_rate": 1.951561542641102e-07,
      "logits/chosen": -0.189265638589859,
      "logits/rejected": -0.18786805868148804,
      "logps/chosen": -1.6608617305755615,
      "logps/rejected": -2.460606098175049,
      "loss": 0.4715,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6608617305755615,
      "rewards/margins": 0.7997443675994873,
      "rewards/rejected": -2.460606098175049,
      "sft_loss": 1.7233469486236572,
      "step": 4135
    },
    {
      "epoch": 2.215755143000502,
      "grad_norm": 10.22155169357895,
      "learning_rate": 1.939231727099806e-07,
      "logits/chosen": -0.34872856736183167,
      "logits/rejected": -0.25761353969573975,
      "logps/chosen": -1.514994740486145,
      "logps/rejected": -2.1406776905059814,
      "loss": 0.4381,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.514994740486145,
      "rewards/margins": 0.6256829500198364,
      "rewards/rejected": -2.1406776905059814,
      "sft_loss": 1.6128156185150146,
      "step": 4140
    },
    {
      "epoch": 2.2184311757819035,
      "grad_norm": 8.166285788125295,
      "learning_rate": 1.926931607145719e-07,
      "logits/chosen": -0.13309597969055176,
      "logits/rejected": 0.01610729657113552,
      "logps/chosen": -1.6787502765655518,
      "logps/rejected": -2.3410792350769043,
      "loss": 0.4649,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.6787502765655518,
      "rewards/margins": 0.6623291373252869,
      "rewards/rejected": -2.3410792350769043,
      "sft_loss": 1.7590233087539673,
      "step": 4145
    },
    {
      "epoch": 2.221107208563305,
      "grad_norm": 6.692670737186019,
      "learning_rate": 1.9146613021146564e-07,
      "logits/chosen": -0.18270069360733032,
      "logits/rejected": -0.07625246793031693,
      "logps/chosen": -1.4268088340759277,
      "logps/rejected": -2.050219774246216,
      "loss": 0.4379,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4268088340759277,
      "rewards/margins": 0.6234109997749329,
      "rewards/rejected": -2.050219774246216,
      "sft_loss": 1.5354491472244263,
      "step": 4150
    },
    {
      "epoch": 2.2237832413447065,
      "grad_norm": 9.108793827152095,
      "learning_rate": 1.9024209310531736e-07,
      "logits/chosen": -0.1617237627506256,
      "logits/rejected": -0.17442257702350616,
      "logps/chosen": -1.5069429874420166,
      "logps/rejected": -2.02146053314209,
      "loss": 0.4658,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5069429874420166,
      "rewards/margins": 0.5145175457000732,
      "rewards/rejected": -2.02146053314209,
      "sft_loss": 1.567132592201233,
      "step": 4155
    },
    {
      "epoch": 2.2264592741261082,
      "grad_norm": 10.438365653518229,
      "learning_rate": 1.890210612717401e-07,
      "logits/chosen": -0.20505666732788086,
      "logits/rejected": -0.040448617190122604,
      "logps/chosen": -1.6469700336456299,
      "logps/rejected": -2.1686878204345703,
      "loss": 0.4657,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6469700336456299,
      "rewards/margins": 0.5217178463935852,
      "rewards/rejected": -2.1686878204345703,
      "sft_loss": 1.7205617427825928,
      "step": 4160
    },
    {
      "epoch": 2.2291353069075095,
      "grad_norm": 11.070803534114043,
      "learning_rate": 1.8780304655719054e-07,
      "logits/chosen": -0.2014208287000656,
      "logits/rejected": -0.06682918220758438,
      "logps/chosen": -1.5332292318344116,
      "logps/rejected": -2.188716173171997,
      "loss": 0.4628,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5332292318344116,
      "rewards/margins": 0.6554868817329407,
      "rewards/rejected": -2.188716173171997,
      "sft_loss": 1.6469694375991821,
      "step": 4165
    },
    {
      "epoch": 2.231811339688911,
      "grad_norm": 9.181331373544161,
      "learning_rate": 1.865880607788523e-07,
      "logits/chosen": -0.09014538675546646,
      "logits/rejected": -0.01346281636506319,
      "logps/chosen": -1.5006886720657349,
      "logps/rejected": -2.145022392272949,
      "loss": 0.4344,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5006886720657349,
      "rewards/margins": 0.6443338394165039,
      "rewards/rejected": -2.145022392272949,
      "sft_loss": 1.5990290641784668,
      "step": 4170
    },
    {
      "epoch": 2.234487372470313,
      "grad_norm": 12.213842457899116,
      "learning_rate": 1.8537611572452316e-07,
      "logits/chosen": -0.219885915517807,
      "logits/rejected": -0.09703179448843002,
      "logps/chosen": -1.518066644668579,
      "logps/rejected": -1.9615545272827148,
      "loss": 0.4939,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.518066644668579,
      "rewards/margins": 0.4434877336025238,
      "rewards/rejected": -1.9615545272827148,
      "sft_loss": 1.5732790231704712,
      "step": 4175
    },
    {
      "epoch": 2.237163405251714,
      "grad_norm": 10.116284053671981,
      "learning_rate": 1.84167223152499e-07,
      "logits/chosen": -0.23928925395011902,
      "logits/rejected": 0.006835061125457287,
      "logps/chosen": -1.4725453853607178,
      "logps/rejected": -2.2228915691375732,
      "loss": 0.4093,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4725453853607178,
      "rewards/margins": 0.7503459453582764,
      "rewards/rejected": -2.2228915691375732,
      "sft_loss": 1.584643006324768,
      "step": 4180
    },
    {
      "epoch": 2.239839438033116,
      "grad_norm": 8.807905961289755,
      "learning_rate": 1.8296139479146112e-07,
      "logits/chosen": -0.27372652292251587,
      "logits/rejected": -0.2470136135816574,
      "logps/chosen": -1.4996545314788818,
      "logps/rejected": -2.1062915325164795,
      "loss": 0.4589,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4996545314788818,
      "rewards/margins": 0.6066369414329529,
      "rewards/rejected": -2.1062915325164795,
      "sft_loss": 1.6074802875518799,
      "step": 4185
    },
    {
      "epoch": 2.2425154708145176,
      "grad_norm": 6.763198406992284,
      "learning_rate": 1.8175864234036132e-07,
      "logits/chosen": -0.11524118483066559,
      "logits/rejected": 0.0006000146386213601,
      "logps/chosen": -1.4346461296081543,
      "logps/rejected": -2.095411539077759,
      "loss": 0.4311,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4346461296081543,
      "rewards/margins": 0.6607652902603149,
      "rewards/rejected": -2.095411539077759,
      "sft_loss": 1.534780502319336,
      "step": 4190
    },
    {
      "epoch": 2.245191503595919,
      "grad_norm": 5.831391069472206,
      "learning_rate": 1.805589774683094e-07,
      "logits/chosen": -0.32675114274024963,
      "logits/rejected": -0.1725514978170395,
      "logps/chosen": -1.5440665483474731,
      "logps/rejected": -2.0298123359680176,
      "loss": 0.4909,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5440665483474731,
      "rewards/margins": 0.48574599623680115,
      "rewards/rejected": -2.0298123359680176,
      "sft_loss": 1.6876633167266846,
      "step": 4195
    },
    {
      "epoch": 2.2478675363773206,
      "grad_norm": 8.354466364764948,
      "learning_rate": 1.79362411814459e-07,
      "logits/chosen": -0.08029486984014511,
      "logits/rejected": -0.09093089401721954,
      "logps/chosen": -1.6234248876571655,
      "logps/rejected": -2.1632299423217773,
      "loss": 0.4929,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6234248876571655,
      "rewards/margins": 0.5398050546646118,
      "rewards/rejected": -2.1632299423217773,
      "sft_loss": 1.6912288665771484,
      "step": 4200
    },
    {
      "epoch": 2.2505435691587223,
      "grad_norm": 6.8242452196900185,
      "learning_rate": 1.7816895698789552e-07,
      "logits/chosen": -0.2887396812438965,
      "logits/rejected": -0.21421535313129425,
      "logps/chosen": -1.4559670686721802,
      "logps/rejected": -2.0635907649993896,
      "loss": 0.4248,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4559670686721802,
      "rewards/margins": 0.607623815536499,
      "rewards/rejected": -2.0635907649993896,
      "sft_loss": 1.5633571147918701,
      "step": 4205
    },
    {
      "epoch": 2.2532196019401236,
      "grad_norm": 5.827910714951243,
      "learning_rate": 1.7697862456752271e-07,
      "logits/chosen": -0.23325955867767334,
      "logits/rejected": -0.10476813465356827,
      "logps/chosen": -1.5497891902923584,
      "logps/rejected": -2.382882595062256,
      "loss": 0.4267,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5497891902923584,
      "rewards/margins": 0.8330934643745422,
      "rewards/rejected": -2.382882595062256,
      "sft_loss": 1.684547781944275,
      "step": 4210
    },
    {
      "epoch": 2.2558956347215253,
      "grad_norm": 9.32355207297127,
      "learning_rate": 1.7579142610195124e-07,
      "logits/chosen": -0.23333308100700378,
      "logits/rejected": -0.07826565951108932,
      "logps/chosen": -1.505821704864502,
      "logps/rejected": -2.1882753372192383,
      "loss": 0.4583,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.505821704864502,
      "rewards/margins": 0.6824537515640259,
      "rewards/rejected": -2.1882753372192383,
      "sft_loss": 1.5673660039901733,
      "step": 4215
    },
    {
      "epoch": 2.258571667502927,
      "grad_norm": 8.015105705559941,
      "learning_rate": 1.7460737310938568e-07,
      "logits/chosen": -0.2862780690193176,
      "logits/rejected": -0.06725447624921799,
      "logps/chosen": -1.5083166360855103,
      "logps/rejected": -2.1370108127593994,
      "loss": 0.4381,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5083166360855103,
      "rewards/margins": 0.6286943554878235,
      "rewards/rejected": -2.1370108127593994,
      "sft_loss": 1.6248115301132202,
      "step": 4220
    },
    {
      "epoch": 2.2612477002843283,
      "grad_norm": 7.8642532134225815,
      "learning_rate": 1.734264770775133e-07,
      "logits/chosen": -0.277448832988739,
      "logits/rejected": -0.07319362461566925,
      "logps/chosen": -1.5987507104873657,
      "logps/rejected": -2.286151170730591,
      "loss": 0.4288,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.5987507104873657,
      "rewards/margins": 0.6874004602432251,
      "rewards/rejected": -2.286151170730591,
      "sft_loss": 1.6796283721923828,
      "step": 4225
    },
    {
      "epoch": 2.26392373306573,
      "grad_norm": 6.92372534786143,
      "learning_rate": 1.7224874946339241e-07,
      "logits/chosen": -0.26496168971061707,
      "logits/rejected": -0.15971460938453674,
      "logps/chosen": -1.5722956657409668,
      "logps/rejected": -2.2052111625671387,
      "loss": 0.4795,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5722956657409668,
      "rewards/margins": 0.6329156160354614,
      "rewards/rejected": -2.2052111625671387,
      "sft_loss": 1.6152551174163818,
      "step": 4230
    },
    {
      "epoch": 2.2665997658471317,
      "grad_norm": 6.258699276893173,
      "learning_rate": 1.7107420169334186e-07,
      "logits/chosen": -0.19774913787841797,
      "logits/rejected": -0.07455114275217056,
      "logps/chosen": -1.5831193923950195,
      "logps/rejected": -2.035914897918701,
      "loss": 0.505,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5831193923950195,
      "rewards/margins": 0.45279568433761597,
      "rewards/rejected": -2.035914897918701,
      "sft_loss": 1.7011892795562744,
      "step": 4235
    },
    {
      "epoch": 2.269275798628533,
      "grad_norm": 7.605306577163764,
      "learning_rate": 1.6990284516282893e-07,
      "logits/chosen": -0.229135662317276,
      "logits/rejected": -0.0910092145204544,
      "logps/chosen": -1.5330448150634766,
      "logps/rejected": -2.0770211219787598,
      "loss": 0.4496,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5330448150634766,
      "rewards/margins": 0.5439764261245728,
      "rewards/rejected": -2.0770211219787598,
      "sft_loss": 1.6638119220733643,
      "step": 4240
    },
    {
      "epoch": 2.2719518314099347,
      "grad_norm": 10.699895417760684,
      "learning_rate": 1.687346912363602e-07,
      "logits/chosen": -0.2965095341205597,
      "logits/rejected": -0.1600067913532257,
      "logps/chosen": -1.6080896854400635,
      "logps/rejected": -2.174994945526123,
      "loss": 0.4808,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6080896854400635,
      "rewards/margins": 0.5669052600860596,
      "rewards/rejected": -2.174994945526123,
      "sft_loss": 1.7067543268203735,
      "step": 4245
    },
    {
      "epoch": 2.2746278641913364,
      "grad_norm": 6.405918147574539,
      "learning_rate": 1.675697512473697e-07,
      "logits/chosen": -0.21542322635650635,
      "logits/rejected": -0.010124053806066513,
      "logps/chosen": -1.629784345626831,
      "logps/rejected": -2.248307943344116,
      "loss": 0.4734,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.629784345626831,
      "rewards/margins": 0.6185236573219299,
      "rewards/rejected": -2.248307943344116,
      "sft_loss": 1.686134696006775,
      "step": 4250
    },
    {
      "epoch": 2.2773038969727377,
      "grad_norm": 11.065224789631493,
      "learning_rate": 1.6640803649811087e-07,
      "logits/chosen": -0.24650788307189941,
      "logits/rejected": 0.009222614578902721,
      "logps/chosen": -1.6138334274291992,
      "logps/rejected": -2.3137295246124268,
      "loss": 0.4455,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.6138334274291992,
      "rewards/margins": 0.6998960375785828,
      "rewards/rejected": -2.3137295246124268,
      "sft_loss": 1.6994917392730713,
      "step": 4255
    },
    {
      "epoch": 2.2799799297541394,
      "grad_norm": 7.584697213644501,
      "learning_rate": 1.6524955825954472e-07,
      "logits/chosen": -0.21565786004066467,
      "logits/rejected": -0.09113094955682755,
      "logps/chosen": -1.461003303527832,
      "logps/rejected": -2.0535426139831543,
      "loss": 0.4331,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.461003303527832,
      "rewards/margins": 0.592539370059967,
      "rewards/rejected": -2.0535426139831543,
      "sft_loss": 1.4886611700057983,
      "step": 4260
    },
    {
      "epoch": 2.282655962535541,
      "grad_norm": 6.263987411806834,
      "learning_rate": 1.6409432777123277e-07,
      "logits/chosen": -0.3257189989089966,
      "logits/rejected": -0.13461880385875702,
      "logps/chosen": -1.522209644317627,
      "logps/rejected": -2.2624077796936035,
      "loss": 0.4186,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.522209644317627,
      "rewards/margins": 0.7401978373527527,
      "rewards/rejected": -2.2624077796936035,
      "sft_loss": 1.6207393407821655,
      "step": 4265
    },
    {
      "epoch": 2.285331995316943,
      "grad_norm": 7.356539401543046,
      "learning_rate": 1.6294235624122577e-07,
      "logits/chosen": -0.12847329676151276,
      "logits/rejected": 0.13997411727905273,
      "logps/chosen": -1.586727261543274,
      "logps/rejected": -2.3713271617889404,
      "loss": 0.4369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.586727261543274,
      "rewards/margins": 0.7845998406410217,
      "rewards/rejected": -2.3713271617889404,
      "sft_loss": 1.6232751607894897,
      "step": 4270
    },
    {
      "epoch": 2.288008028098344,
      "grad_norm": 8.907757600140817,
      "learning_rate": 1.6179365484595697e-07,
      "logits/chosen": -0.17414642870426178,
      "logits/rejected": -0.06463111937046051,
      "logps/chosen": -1.633993148803711,
      "logps/rejected": -2.227522611618042,
      "loss": 0.4818,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.633993148803711,
      "rewards/margins": 0.5935293436050415,
      "rewards/rejected": -2.227522611618042,
      "sft_loss": 1.7294479608535767,
      "step": 4275
    },
    {
      "epoch": 2.290684060879746,
      "grad_norm": 11.731433072211523,
      "learning_rate": 1.60648234730132e-07,
      "logits/chosen": -0.2536371648311615,
      "logits/rejected": -0.1423775851726532,
      "logps/chosen": -1.4991668462753296,
      "logps/rejected": -2.1331608295440674,
      "loss": 0.4366,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4991668462753296,
      "rewards/margins": 0.6339941024780273,
      "rewards/rejected": -2.1331608295440674,
      "sft_loss": 1.607783317565918,
      "step": 4280
    },
    {
      "epoch": 2.293360093661147,
      "grad_norm": 12.406651356740806,
      "learning_rate": 1.595061070066222e-07,
      "logits/chosen": -0.13887982070446014,
      "logits/rejected": -0.1457427442073822,
      "logps/chosen": -1.4876251220703125,
      "logps/rejected": -2.082301139831543,
      "loss": 0.453,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4876251220703125,
      "rewards/margins": 0.59467613697052,
      "rewards/rejected": -2.082301139831543,
      "sft_loss": 1.6114146709442139,
      "step": 4285
    },
    {
      "epoch": 2.296036126442549,
      "grad_norm": 8.930338489655787,
      "learning_rate": 1.5836728275635542e-07,
      "logits/chosen": -0.27472439408302307,
      "logits/rejected": -0.09401975572109222,
      "logps/chosen": -1.6199219226837158,
      "logps/rejected": -2.2362000942230225,
      "loss": 0.4881,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.6199219226837158,
      "rewards/margins": 0.616278350353241,
      "rewards/rejected": -2.2362000942230225,
      "sft_loss": 1.6935663223266602,
      "step": 4290
    },
    {
      "epoch": 2.2987121592239506,
      "grad_norm": 10.706568122216572,
      "learning_rate": 1.5723177302820984e-07,
      "logits/chosen": -0.24929042160511017,
      "logits/rejected": -0.18465907871723175,
      "logps/chosen": -1.5443518161773682,
      "logps/rejected": -2.1019937992095947,
      "loss": 0.4599,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5443518161773682,
      "rewards/margins": 0.5576421022415161,
      "rewards/rejected": -2.1019937992095947,
      "sft_loss": 1.6194578409194946,
      "step": 4295
    },
    {
      "epoch": 2.3013881920053523,
      "grad_norm": 9.638950157766153,
      "learning_rate": 1.5609958883890544e-07,
      "logits/chosen": -0.23110198974609375,
      "logits/rejected": -0.08799050003290176,
      "logps/chosen": -1.543038010597229,
      "logps/rejected": -2.047542095184326,
      "loss": 0.4726,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.543038010597229,
      "rewards/margins": 0.5045040249824524,
      "rewards/rejected": -2.047542095184326,
      "sft_loss": 1.6039879322052002,
      "step": 4300
    },
    {
      "epoch": 2.3040642247867535,
      "grad_norm": 12.648365762769789,
      "learning_rate": 1.5497074117289865e-07,
      "logits/chosen": -0.3018723428249359,
      "logits/rejected": -0.16592738032341003,
      "logps/chosen": -1.4970450401306152,
      "logps/rejected": -2.2717602252960205,
      "loss": 0.4212,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4970450401306152,
      "rewards/margins": 0.7747151851654053,
      "rewards/rejected": -2.2717602252960205,
      "sft_loss": 1.6478235721588135,
      "step": 4305
    },
    {
      "epoch": 2.3067402575681553,
      "grad_norm": 6.599885818260292,
      "learning_rate": 1.5384524098227402e-07,
      "logits/chosen": -0.2461796998977661,
      "logits/rejected": -0.033960431814193726,
      "logps/chosen": -1.504396677017212,
      "logps/rejected": -2.293781280517578,
      "loss": 0.3995,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.504396677017212,
      "rewards/margins": 0.7893846035003662,
      "rewards/rejected": -2.293781280517578,
      "sft_loss": 1.5998690128326416,
      "step": 4310
    },
    {
      "epoch": 2.3094162903495565,
      "grad_norm": 13.023271021475802,
      "learning_rate": 1.5272309918663974e-07,
      "logits/chosen": -0.22546645998954773,
      "logits/rejected": -0.05487429350614548,
      "logps/chosen": -1.58822500705719,
      "logps/rejected": -2.127042293548584,
      "loss": 0.4672,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.58822500705719,
      "rewards/margins": 0.5388172268867493,
      "rewards/rejected": -2.127042293548584,
      "sft_loss": 1.7143867015838623,
      "step": 4315
    },
    {
      "epoch": 2.3120923231309582,
      "grad_norm": 12.84040076737523,
      "learning_rate": 1.516043266730201e-07,
      "logits/chosen": -0.24874725937843323,
      "logits/rejected": -0.10095198452472687,
      "logps/chosen": -1.6159508228302002,
      "logps/rejected": -2.2336201667785645,
      "loss": 0.4561,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6159508228302002,
      "rewards/margins": 0.6176694631576538,
      "rewards/rejected": -2.2336201667785645,
      "sft_loss": 1.6980879306793213,
      "step": 4320
    },
    {
      "epoch": 2.31476835591236,
      "grad_norm": 9.003986896162802,
      "learning_rate": 1.504889342957512e-07,
      "logits/chosen": -0.2251073122024536,
      "logits/rejected": -0.04803949221968651,
      "logps/chosen": -1.5959573984146118,
      "logps/rejected": -2.325037717819214,
      "loss": 0.4697,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5959573984146118,
      "rewards/margins": 0.729080319404602,
      "rewards/rejected": -2.325037717819214,
      "sft_loss": 1.7491687536239624,
      "step": 4325
    },
    {
      "epoch": 2.3174443886937617,
      "grad_norm": 10.155414638946343,
      "learning_rate": 1.4937693287637453e-07,
      "logits/chosen": -0.22241003811359406,
      "logits/rejected": -0.09521248191595078,
      "logps/chosen": -1.6965595483779907,
      "logps/rejected": -2.3352200984954834,
      "loss": 0.5169,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6965595483779907,
      "rewards/margins": 0.6386607885360718,
      "rewards/rejected": -2.3352200984954834,
      "sft_loss": 1.7339941263198853,
      "step": 4330
    },
    {
      "epoch": 2.320120421475163,
      "grad_norm": 6.532500154575078,
      "learning_rate": 1.4826833320353305e-07,
      "logits/chosen": -0.18349921703338623,
      "logits/rejected": -0.10400259494781494,
      "logps/chosen": -1.583504557609558,
      "logps/rejected": -2.2146687507629395,
      "loss": 0.4526,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.583504557609558,
      "rewards/margins": 0.6311641335487366,
      "rewards/rejected": -2.2146687507629395,
      "sft_loss": 1.6056795120239258,
      "step": 4335
    },
    {
      "epoch": 2.3227964542565647,
      "grad_norm": 9.741375398588128,
      "learning_rate": 1.4716314603286528e-07,
      "logits/chosen": -0.239861398935318,
      "logits/rejected": -0.054199181497097015,
      "logps/chosen": -1.4949240684509277,
      "logps/rejected": -2.2963128089904785,
      "loss": 0.4181,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4949240684509277,
      "rewards/margins": 0.8013887405395508,
      "rewards/rejected": -2.2963128089904785,
      "sft_loss": 1.6196388006210327,
      "step": 4340
    },
    {
      "epoch": 2.3254724870379664,
      "grad_norm": 20.376394353403786,
      "learning_rate": 1.4606138208690233e-07,
      "logits/chosen": -0.28271156549453735,
      "logits/rejected": -0.20126430690288544,
      "logps/chosen": -1.7372280359268188,
      "logps/rejected": -2.233088970184326,
      "loss": 0.5249,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.7372280359268188,
      "rewards/margins": 0.49586066603660583,
      "rewards/rejected": -2.233088970184326,
      "sft_loss": 1.8056633472442627,
      "step": 4345
    },
    {
      "epoch": 2.3281485198193677,
      "grad_norm": 8.267825898405084,
      "learning_rate": 1.4496305205496251e-07,
      "logits/chosen": -0.19051404297351837,
      "logits/rejected": -0.11613442748785019,
      "logps/chosen": -1.6146233081817627,
      "logps/rejected": -2.352165699005127,
      "loss": 0.4207,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6146233081817627,
      "rewards/margins": 0.737542450428009,
      "rewards/rejected": -2.352165699005127,
      "sft_loss": 1.7026243209838867,
      "step": 4350
    },
    {
      "epoch": 2.3308245526007694,
      "grad_norm": 6.724335492222933,
      "learning_rate": 1.4386816659304895e-07,
      "logits/chosen": -0.26722902059555054,
      "logits/rejected": -0.1480477899312973,
      "logps/chosen": -1.5895460844039917,
      "logps/rejected": -2.1785476207733154,
      "loss": 0.4478,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5895460844039917,
      "rewards/margins": 0.5890012979507446,
      "rewards/rejected": -2.1785476207733154,
      "sft_loss": 1.6915168762207031,
      "step": 4355
    },
    {
      "epoch": 2.333500585382171,
      "grad_norm": 7.925352571352285,
      "learning_rate": 1.4277673632374492e-07,
      "logits/chosen": -0.30871835350990295,
      "logits/rejected": -0.07422409951686859,
      "logps/chosen": -1.6400692462921143,
      "logps/rejected": -2.2754616737365723,
      "loss": 0.4412,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6400692462921143,
      "rewards/margins": 0.6353921890258789,
      "rewards/rejected": -2.2754616737365723,
      "sft_loss": 1.7322028875350952,
      "step": 4360
    },
    {
      "epoch": 2.3361766181635724,
      "grad_norm": 7.6566753761813615,
      "learning_rate": 1.416887718361119e-07,
      "logits/chosen": -0.14006388187408447,
      "logits/rejected": -0.13663342595100403,
      "logps/chosen": -1.5292609930038452,
      "logps/rejected": -2.09344220161438,
      "loss": 0.4701,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5292609930038452,
      "rewards/margins": 0.5641810297966003,
      "rewards/rejected": -2.09344220161438,
      "sft_loss": 1.6062122583389282,
      "step": 4365
    },
    {
      "epoch": 2.338852650944974,
      "grad_norm": 9.318812779298241,
      "learning_rate": 1.406042836855859e-07,
      "logits/chosen": -0.17240700125694275,
      "logits/rejected": -0.024434376507997513,
      "logps/chosen": -1.409942626953125,
      "logps/rejected": -2.1142661571502686,
      "loss": 0.4084,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.409942626953125,
      "rewards/margins": 0.7043234705924988,
      "rewards/rejected": -2.1142661571502686,
      "sft_loss": 1.484179973602295,
      "step": 4370
    },
    {
      "epoch": 2.341528683726376,
      "grad_norm": 8.427360444991452,
      "learning_rate": 1.3952328239387595e-07,
      "logits/chosen": -0.30821970105171204,
      "logits/rejected": -0.09003923833370209,
      "logps/chosen": -1.610558271408081,
      "logps/rejected": -2.268453598022461,
      "loss": 0.4802,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.610558271408081,
      "rewards/margins": 0.6578952074050903,
      "rewards/rejected": -2.268453598022461,
      "sft_loss": 1.7595865726470947,
      "step": 4375
    },
    {
      "epoch": 2.344204716507777,
      "grad_norm": 7.140929143265957,
      "learning_rate": 1.3844577844886109e-07,
      "logits/chosen": -0.26558440923690796,
      "logits/rejected": -0.05220402404665947,
      "logps/chosen": -1.6598186492919922,
      "logps/rejected": -2.2687034606933594,
      "loss": 0.4887,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.6598186492919922,
      "rewards/margins": 0.6088845133781433,
      "rewards/rejected": -2.2687034606933594,
      "sft_loss": 1.7320663928985596,
      "step": 4380
    },
    {
      "epoch": 2.346880749289179,
      "grad_norm": 13.276671883952789,
      "learning_rate": 1.3737178230448955e-07,
      "logits/chosen": -0.3073621690273285,
      "logits/rejected": -0.14894798398017883,
      "logps/chosen": -1.6358146667480469,
      "logps/rejected": -2.2647910118103027,
      "loss": 0.4647,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6358146667480469,
      "rewards/margins": 0.6289765238761902,
      "rewards/rejected": -2.2647910118103027,
      "sft_loss": 1.7382218837738037,
      "step": 4385
    },
    {
      "epoch": 2.3495567820705805,
      "grad_norm": 5.620089072897995,
      "learning_rate": 1.363013043806764e-07,
      "logits/chosen": -0.25026455521583557,
      "logits/rejected": -0.10177797079086304,
      "logps/chosen": -1.5480822324752808,
      "logps/rejected": -2.1520581245422363,
      "loss": 0.4538,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5480822324752808,
      "rewards/margins": 0.6039758920669556,
      "rewards/rejected": -2.1520581245422363,
      "sft_loss": 1.6605409383773804,
      "step": 4390
    },
    {
      "epoch": 2.3522328148519818,
      "grad_norm": 8.59565820868914,
      "learning_rate": 1.352343550632034e-07,
      "logits/chosen": -0.18619614839553833,
      "logits/rejected": -0.018567675724625587,
      "logps/chosen": -1.5407416820526123,
      "logps/rejected": -2.2864482402801514,
      "loss": 0.4455,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5407416820526123,
      "rewards/margins": 0.7457064986228943,
      "rewards/rejected": -2.2864482402801514,
      "sft_loss": 1.6421838998794556,
      "step": 4395
    },
    {
      "epoch": 2.3549088476333835,
      "grad_norm": 6.951569508645217,
      "learning_rate": 1.3417094470361722e-07,
      "logits/chosen": -0.2967742383480072,
      "logits/rejected": -0.1319170445203781,
      "logps/chosen": -1.5985156297683716,
      "logps/rejected": -2.249882221221924,
      "loss": 0.4418,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5985156297683716,
      "rewards/margins": 0.651366651058197,
      "rewards/rejected": -2.249882221221924,
      "sft_loss": 1.6927413940429688,
      "step": 4400
    },
    {
      "epoch": 2.3549088476333835,
      "eval_logits/chosen": 0.1534985452890396,
      "eval_logits/rejected": 0.2581808865070343,
      "eval_logps/chosen": -1.6394926309585571,
      "eval_logps/rejected": -2.227719783782959,
      "eval_loss": 0.49088484048843384,
      "eval_rewards/accuracies": 0.6735904812812805,
      "eval_rewards/chosen": -1.6394926309585571,
      "eval_rewards/margins": 0.5882269144058228,
      "eval_rewards/rejected": -2.227719783782959,
      "eval_runtime": 43.4714,
      "eval_samples_per_second": 30.94,
      "eval_sft_loss": 1.7121342420578003,
      "eval_steps_per_second": 7.752,
      "step": 4400
    },
    {
      "epoch": 2.357584880414785,
      "grad_norm": 8.04651481113309,
      "learning_rate": 1.3311108361913015e-07,
      "logits/chosen": -0.28536808490753174,
      "logits/rejected": -0.24391081929206848,
      "logps/chosen": -1.5283732414245605,
      "logps/rejected": -2.0771474838256836,
      "loss": 0.4629,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5283732414245605,
      "rewards/margins": 0.548774242401123,
      "rewards/rejected": -2.0771474838256836,
      "sft_loss": 1.613776445388794,
      "step": 4405
    },
    {
      "epoch": 2.3602609131961865,
      "grad_norm": 7.95291662888706,
      "learning_rate": 1.3205478209251874e-07,
      "logits/chosen": -0.23777012526988983,
      "logits/rejected": -0.1785149872303009,
      "logps/chosen": -1.6985456943511963,
      "logps/rejected": -2.46702241897583,
      "loss": 0.4479,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6985456943511963,
      "rewards/margins": 0.7684768438339233,
      "rewards/rejected": -2.46702241897583,
      "sft_loss": 1.7738962173461914,
      "step": 4410
    },
    {
      "epoch": 2.362936945977588,
      "grad_norm": 7.746688363671087,
      "learning_rate": 1.310020503720254e-07,
      "logits/chosen": -0.23160281777381897,
      "logits/rejected": -0.04335252568125725,
      "logps/chosen": -1.609033226966858,
      "logps/rejected": -2.23053240776062,
      "loss": 0.4682,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.609033226966858,
      "rewards/margins": 0.6214991807937622,
      "rewards/rejected": -2.23053240776062,
      "sft_loss": 1.676290512084961,
      "step": 4415
    },
    {
      "epoch": 2.36561297875899,
      "grad_norm": 10.610213280300115,
      "learning_rate": 1.2995289867125752e-07,
      "logits/chosen": -0.2662007212638855,
      "logits/rejected": -0.17510481178760529,
      "logps/chosen": -1.6208078861236572,
      "logps/rejected": -2.1000378131866455,
      "loss": 0.4793,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6208078861236572,
      "rewards/margins": 0.4792299270629883,
      "rewards/rejected": -2.1000378131866455,
      "sft_loss": 1.692181944847107,
      "step": 4420
    },
    {
      "epoch": 2.368289011540391,
      "grad_norm": 8.936766324818056,
      "learning_rate": 1.2890733716908986e-07,
      "logits/chosen": -0.23490257561206818,
      "logits/rejected": -0.11895327270030975,
      "logps/chosen": -1.4317034482955933,
      "logps/rejected": -2.0047073364257812,
      "loss": 0.4219,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4317034482955933,
      "rewards/margins": 0.573003888130188,
      "rewards/rejected": -2.0047073364257812,
      "sft_loss": 1.589618444442749,
      "step": 4425
    },
    {
      "epoch": 2.370965044321793,
      "grad_norm": 7.21487936412045,
      "learning_rate": 1.2786537600956454e-07,
      "logits/chosen": -0.25477251410484314,
      "logits/rejected": -0.06471340358257294,
      "logps/chosen": -1.6068341732025146,
      "logps/rejected": -2.2490625381469727,
      "loss": 0.4545,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6068341732025146,
      "rewards/margins": 0.6422284841537476,
      "rewards/rejected": -2.2490625381469727,
      "sft_loss": 1.6686866283416748,
      "step": 4430
    },
    {
      "epoch": 2.3736410771031946,
      "grad_norm": 6.72950333850293,
      "learning_rate": 1.268270253017933e-07,
      "logits/chosen": -0.32577234506607056,
      "logits/rejected": -0.0894775241613388,
      "logps/chosen": -1.5162084102630615,
      "logps/rejected": -2.182002544403076,
      "loss": 0.4162,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5162084102630615,
      "rewards/margins": 0.6657941341400146,
      "rewards/rejected": -2.182002544403076,
      "sft_loss": 1.6630744934082031,
      "step": 4435
    },
    {
      "epoch": 2.376317109884596,
      "grad_norm": 9.555770264870109,
      "learning_rate": 1.257922951198591e-07,
      "logits/chosen": -0.33920782804489136,
      "logits/rejected": -0.09382447600364685,
      "logps/chosen": -1.5818603038787842,
      "logps/rejected": -2.1809945106506348,
      "loss": 0.4651,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5818603038787842,
      "rewards/margins": 0.5991338491439819,
      "rewards/rejected": -2.1809945106506348,
      "sft_loss": 1.6767442226409912,
      "step": 4440
    },
    {
      "epoch": 2.3789931426659976,
      "grad_norm": 8.69159399901978,
      "learning_rate": 1.24761195502719e-07,
      "logits/chosen": -0.2876212000846863,
      "logits/rejected": -0.08851093053817749,
      "logps/chosen": -1.5860512256622314,
      "logps/rejected": -2.099834442138672,
      "loss": 0.4902,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.5860512256622314,
      "rewards/margins": 0.5137835144996643,
      "rewards/rejected": -2.099834442138672,
      "sft_loss": 1.675398588180542,
      "step": 4445
    },
    {
      "epoch": 2.3816691754473993,
      "grad_norm": 13.63365552934808,
      "learning_rate": 1.2373373645410573e-07,
      "logits/chosen": -0.23422542214393616,
      "logits/rejected": -0.09559222310781479,
      "logps/chosen": -1.6235287189483643,
      "logps/rejected": -2.3158679008483887,
      "loss": 0.4842,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.6235287189483643,
      "rewards/margins": 0.692339301109314,
      "rewards/rejected": -2.3158679008483887,
      "sft_loss": 1.6855251789093018,
      "step": 4450
    },
    {
      "epoch": 2.384345208228801,
      "grad_norm": 12.73541327236651,
      "learning_rate": 1.2270992794243175e-07,
      "logits/chosen": -0.2981303334236145,
      "logits/rejected": -0.18724291026592255,
      "logps/chosen": -1.5792548656463623,
      "logps/rejected": -2.230262517929077,
      "loss": 0.4532,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5792548656463623,
      "rewards/margins": 0.6510077118873596,
      "rewards/rejected": -2.230262517929077,
      "sft_loss": 1.7070000171661377,
      "step": 4455
    },
    {
      "epoch": 2.3870212410102023,
      "grad_norm": 9.25717711299543,
      "learning_rate": 1.2168977990069147e-07,
      "logits/chosen": -0.33461794257164,
      "logits/rejected": -0.09690986573696136,
      "logps/chosen": -1.5364824533462524,
      "logps/rejected": -2.3159992694854736,
      "loss": 0.4211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5364824533462524,
      "rewards/margins": 0.7795166969299316,
      "rewards/rejected": -2.3159992694854736,
      "sft_loss": 1.691641092300415,
      "step": 4460
    },
    {
      "epoch": 2.389697273791604,
      "grad_norm": 11.18028000847737,
      "learning_rate": 1.206733022263659e-07,
      "logits/chosen": -0.3134737014770508,
      "logits/rejected": -0.10241047292947769,
      "logps/chosen": -1.6970819234848022,
      "logps/rejected": -2.298682451248169,
      "loss": 0.5081,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6970819234848022,
      "rewards/margins": 0.6016003489494324,
      "rewards/rejected": -2.298682451248169,
      "sft_loss": 1.7689815759658813,
      "step": 4465
    },
    {
      "epoch": 2.3923733065730053,
      "grad_norm": 7.14924392499626,
      "learning_rate": 1.1966050478132572e-07,
      "logits/chosen": -0.14032816886901855,
      "logits/rejected": -0.053854428231716156,
      "logps/chosen": -1.4425265789031982,
      "logps/rejected": -2.010845184326172,
      "loss": 0.4676,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4425265789031982,
      "rewards/margins": 0.568318784236908,
      "rewards/rejected": -2.010845184326172,
      "sft_loss": 1.5330469608306885,
      "step": 4470
    },
    {
      "epoch": 2.395049339354407,
      "grad_norm": 14.295657293369166,
      "learning_rate": 1.1865139739173635e-07,
      "logits/chosen": -0.24750380218029022,
      "logits/rejected": -0.003714990569278598,
      "logps/chosen": -1.5968362092971802,
      "logps/rejected": -2.2002665996551514,
      "loss": 0.4447,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5968362092971802,
      "rewards/margins": 0.6034305691719055,
      "rewards/rejected": -2.2002665996551514,
      "sft_loss": 1.6557786464691162,
      "step": 4475
    },
    {
      "epoch": 2.3977253721358087,
      "grad_norm": 13.346358629047302,
      "learning_rate": 1.1764598984796187e-07,
      "logits/chosen": -0.24167075753211975,
      "logits/rejected": -0.14121203124523163,
      "logps/chosen": -1.4179035425186157,
      "logps/rejected": -1.9109907150268555,
      "loss": 0.4614,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4179035425186157,
      "rewards/margins": 0.49308720231056213,
      "rewards/rejected": -1.9109907150268555,
      "sft_loss": 1.5340925455093384,
      "step": 4480
    },
    {
      "epoch": 2.4004014049172104,
      "grad_norm": 8.027036857562305,
      "learning_rate": 1.1664429190447095e-07,
      "logits/chosen": -0.21096546947956085,
      "logits/rejected": -0.11151149123907089,
      "logps/chosen": -1.5817596912384033,
      "logps/rejected": -2.2323250770568848,
      "loss": 0.4462,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5817596912384033,
      "rewards/margins": 0.650565505027771,
      "rewards/rejected": -2.2323250770568848,
      "sft_loss": 1.6181080341339111,
      "step": 4485
    },
    {
      "epoch": 2.4030774376986117,
      "grad_norm": 9.416302960957523,
      "learning_rate": 1.1564631327974122e-07,
      "logits/chosen": -0.2919323444366455,
      "logits/rejected": -0.05769073963165283,
      "logps/chosen": -1.5122710466384888,
      "logps/rejected": -2.2569687366485596,
      "loss": 0.4156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5122710466384888,
      "rewards/margins": 0.7446975111961365,
      "rewards/rejected": -2.2569687366485596,
      "sft_loss": 1.627769112586975,
      "step": 4490
    },
    {
      "epoch": 2.4057534704800134,
      "grad_norm": 12.782013253866758,
      "learning_rate": 1.1465206365616587e-07,
      "logits/chosen": -0.33929866552352905,
      "logits/rejected": -0.11169986426830292,
      "logps/chosen": -1.5895986557006836,
      "logps/rejected": -2.131399631500244,
      "loss": 0.4571,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5895986557006836,
      "rewards/margins": 0.5418012142181396,
      "rewards/rejected": -2.131399631500244,
      "sft_loss": 1.735253930091858,
      "step": 4495
    },
    {
      "epoch": 2.408429503261415,
      "grad_norm": 9.718553496516888,
      "learning_rate": 1.1366155267995887e-07,
      "logits/chosen": -0.15040044486522675,
      "logits/rejected": -0.14154385030269623,
      "logps/chosen": -1.5411628484725952,
      "logps/rejected": -2.1179873943328857,
      "loss": 0.4453,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5411628484725952,
      "rewards/margins": 0.5768247246742249,
      "rewards/rejected": -2.1179873943328857,
      "sft_loss": 1.6429777145385742,
      "step": 4500
    },
    {
      "epoch": 2.4111055360428164,
      "grad_norm": 6.6985265123131486,
      "learning_rate": 1.1267478996106228e-07,
      "logits/chosen": -0.2672615945339203,
      "logits/rejected": -0.09151863306760788,
      "logps/chosen": -1.5106264352798462,
      "logps/rejected": -2.1163084506988525,
      "loss": 0.4391,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5106264352798462,
      "rewards/margins": 0.6056820154190063,
      "rewards/rejected": -2.1163084506988525,
      "sft_loss": 1.5961272716522217,
      "step": 4505
    },
    {
      "epoch": 2.413781568824218,
      "grad_norm": 7.265679258943073,
      "learning_rate": 1.116917850730521e-07,
      "logits/chosen": -0.2920701205730438,
      "logits/rejected": -0.13715091347694397,
      "logps/chosen": -1.5710780620574951,
      "logps/rejected": -2.125671863555908,
      "loss": 0.4692,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5710780620574951,
      "rewards/margins": 0.5545935034751892,
      "rewards/rejected": -2.125671863555908,
      "sft_loss": 1.6058059930801392,
      "step": 4510
    },
    {
      "epoch": 2.41645760160562,
      "grad_norm": 7.190580914210609,
      "learning_rate": 1.1071254755304637e-07,
      "logits/chosen": -0.2794167399406433,
      "logits/rejected": -0.1814873069524765,
      "logps/chosen": -1.5710397958755493,
      "logps/rejected": -2.111194372177124,
      "loss": 0.4604,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5710397958755493,
      "rewards/margins": 0.540154755115509,
      "rewards/rejected": -2.111194372177124,
      "sft_loss": 1.6599490642547607,
      "step": 4515
    },
    {
      "epoch": 2.419133634387021,
      "grad_norm": 10.670685800851055,
      "learning_rate": 1.0973708690161143e-07,
      "logits/chosen": -0.22668643295764923,
      "logits/rejected": -0.11982636153697968,
      "logps/chosen": -1.5077179670333862,
      "logps/rejected": -2.278191328048706,
      "loss": 0.4061,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5077179670333862,
      "rewards/margins": 0.7704734802246094,
      "rewards/rejected": -2.278191328048706,
      "sft_loss": 1.5749647617340088,
      "step": 4520
    },
    {
      "epoch": 2.421809667168423,
      "grad_norm": 9.584274557121692,
      "learning_rate": 1.0876541258267119e-07,
      "logits/chosen": -0.33840879797935486,
      "logits/rejected": -0.14164835214614868,
      "logps/chosen": -1.6478707790374756,
      "logps/rejected": -2.3922953605651855,
      "loss": 0.454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6478707790374756,
      "rewards/margins": 0.7444245219230652,
      "rewards/rejected": -2.3922953605651855,
      "sft_loss": 1.7673391103744507,
      "step": 4525
    },
    {
      "epoch": 2.4244856999498245,
      "grad_norm": 9.0319511589624,
      "learning_rate": 1.0779753402341379e-07,
      "logits/chosen": -0.27057284116744995,
      "logits/rejected": -0.16721342504024506,
      "logps/chosen": -1.5868384838104248,
      "logps/rejected": -2.053842544555664,
      "loss": 0.4918,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5868384838104248,
      "rewards/margins": 0.4670039713382721,
      "rewards/rejected": -2.053842544555664,
      "sft_loss": 1.639451026916504,
      "step": 4530
    },
    {
      "epoch": 2.427161732731226,
      "grad_norm": 10.014783939770096,
      "learning_rate": 1.0683346061420157e-07,
      "logits/chosen": -0.13392801582813263,
      "logits/rejected": -0.009936687536537647,
      "logps/chosen": -1.5058220624923706,
      "logps/rejected": -2.101733446121216,
      "loss": 0.463,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5058220624923706,
      "rewards/margins": 0.5959112644195557,
      "rewards/rejected": -2.101733446121216,
      "sft_loss": 1.6472076177597046,
      "step": 4535
    },
    {
      "epoch": 2.4298377655126275,
      "grad_norm": 5.789686194060652,
      "learning_rate": 1.0587320170847874e-07,
      "logits/chosen": -0.1509685218334198,
      "logits/rejected": -0.054625581949949265,
      "logps/chosen": -1.4414759874343872,
      "logps/rejected": -2.034996747970581,
      "loss": 0.4576,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4414759874343872,
      "rewards/margins": 0.5935209393501282,
      "rewards/rejected": -2.034996747970581,
      "sft_loss": 1.5240309238433838,
      "step": 4540
    },
    {
      "epoch": 2.4325137982940293,
      "grad_norm": 8.089792084934379,
      "learning_rate": 1.0491676662268156e-07,
      "logits/chosen": -0.1864078938961029,
      "logits/rejected": -0.0398380383849144,
      "logps/chosen": -1.5171597003936768,
      "logps/rejected": -2.1239724159240723,
      "loss": 0.4606,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5171597003936768,
      "rewards/margins": 0.6068126559257507,
      "rewards/rejected": -2.1239724159240723,
      "sft_loss": 1.605342149734497,
      "step": 4545
    },
    {
      "epoch": 2.4351898310754305,
      "grad_norm": 7.886331372210549,
      "learning_rate": 1.0396416463614732e-07,
      "logits/chosen": -0.28180861473083496,
      "logits/rejected": -0.1611512005329132,
      "logps/chosen": -1.4591307640075684,
      "logps/rejected": -2.0682687759399414,
      "loss": 0.4343,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4591307640075684,
      "rewards/margins": 0.6091380715370178,
      "rewards/rejected": -2.0682687759399414,
      "sft_loss": 1.5430991649627686,
      "step": 4550
    },
    {
      "epoch": 2.4378658638568322,
      "grad_norm": 7.17269276476902,
      "learning_rate": 1.0301540499102479e-07,
      "logits/chosen": -0.21525149047374725,
      "logits/rejected": -0.13745595514774323,
      "logps/chosen": -1.6477893590927124,
      "logps/rejected": -2.127769708633423,
      "loss": 0.479,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6477893590927124,
      "rewards/margins": 0.47998008131980896,
      "rewards/rejected": -2.127769708633423,
      "sft_loss": 1.786193609237671,
      "step": 4555
    },
    {
      "epoch": 2.440541896638234,
      "grad_norm": 6.662430669419078,
      "learning_rate": 1.0207049689218405e-07,
      "logits/chosen": -0.28460606932640076,
      "logits/rejected": -0.08241887390613556,
      "logps/chosen": -1.5029640197753906,
      "logps/rejected": -2.1092498302459717,
      "loss": 0.49,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5029640197753906,
      "rewards/margins": 0.6062856912612915,
      "rewards/rejected": -2.1092498302459717,
      "sft_loss": 1.5308010578155518,
      "step": 4560
    },
    {
      "epoch": 2.4432179294196352,
      "grad_norm": 8.137985723376088,
      "learning_rate": 1.0112944950712782e-07,
      "logits/chosen": -0.207010418176651,
      "logits/rejected": -0.09890707582235336,
      "logps/chosen": -1.6632816791534424,
      "logps/rejected": -2.2407917976379395,
      "loss": 0.4693,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6632816791534424,
      "rewards/margins": 0.5775102376937866,
      "rewards/rejected": -2.2407917976379395,
      "sft_loss": 1.731000304222107,
      "step": 4565
    },
    {
      "epoch": 2.445893962201037,
      "grad_norm": 13.854076075939032,
      "learning_rate": 1.0019227196590174e-07,
      "logits/chosen": -0.15999329090118408,
      "logits/rejected": 0.00819392316043377,
      "logps/chosen": -1.517242431640625,
      "logps/rejected": -2.212289333343506,
      "loss": 0.4323,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.517242431640625,
      "rewards/margins": 0.6950467228889465,
      "rewards/rejected": -2.212289333343506,
      "sft_loss": 1.5932613611221313,
      "step": 4570
    },
    {
      "epoch": 2.4485699949824387,
      "grad_norm": 11.39520887739819,
      "learning_rate": 9.925897336100664e-08,
      "logits/chosen": -0.1302533894777298,
      "logits/rejected": -0.06674132496118546,
      "logps/chosen": -1.5052393674850464,
      "logps/rejected": -2.0828909873962402,
      "loss": 0.4633,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5052393674850464,
      "rewards/margins": 0.5776516199111938,
      "rewards/rejected": -2.0828909873962402,
      "sft_loss": 1.5766162872314453,
      "step": 4575
    },
    {
      "epoch": 2.45124602776384,
      "grad_norm": 7.304941995094994,
      "learning_rate": 9.832956274730946e-08,
      "logits/chosen": -0.200159952044487,
      "logits/rejected": -0.14625905454158783,
      "logps/chosen": -1.4850728511810303,
      "logps/rejected": -2.0518252849578857,
      "loss": 0.4484,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4850728511810303,
      "rewards/margins": 0.566752552986145,
      "rewards/rejected": -2.0518252849578857,
      "sft_loss": 1.6040903329849243,
      "step": 4580
    },
    {
      "epoch": 2.4539220605452416,
      "grad_norm": 8.850165611803599,
      "learning_rate": 9.740404914195633e-08,
      "logits/chosen": -0.2055172473192215,
      "logits/rejected": -0.04428768530488014,
      "logps/chosen": -1.6074609756469727,
      "logps/rejected": -2.1912786960601807,
      "loss": 0.5067,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6074609756469727,
      "rewards/margins": 0.5838177800178528,
      "rewards/rejected": -2.1912786960601807,
      "sft_loss": 1.7369712591171265,
      "step": 4585
    },
    {
      "epoch": 2.4565980933266434,
      "grad_norm": 6.256203184747748,
      "learning_rate": 9.648244152428392e-08,
      "logits/chosen": -0.3060024380683899,
      "logits/rejected": -0.11156409978866577,
      "logps/chosen": -1.4236325025558472,
      "logps/rejected": -1.9613687992095947,
      "loss": 0.4619,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4236325025558472,
      "rewards/margins": 0.5377359986305237,
      "rewards/rejected": -1.9613687992095947,
      "sft_loss": 1.5441299676895142,
      "step": 4590
    },
    {
      "epoch": 2.4592741261080446,
      "grad_norm": 12.088895012902688,
      "learning_rate": 9.556474883573379e-08,
      "logits/chosen": -0.25038835406303406,
      "logits/rejected": -0.11709809303283691,
      "logps/chosen": -1.5372766256332397,
      "logps/rejected": -2.185891628265381,
      "loss": 0.4618,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5372766256332397,
      "rewards/margins": 0.6486150026321411,
      "rewards/rejected": -2.185891628265381,
      "sft_loss": 1.6226387023925781,
      "step": 4595
    },
    {
      "epoch": 2.4619501588894463,
      "grad_norm": 8.580375617672612,
      "learning_rate": 9.465097997976412e-08,
      "logits/chosen": -0.2508789002895355,
      "logits/rejected": 0.03930927440524101,
      "logps/chosen": -1.5669784545898438,
      "logps/rejected": -2.2951834201812744,
      "loss": 0.4218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5669784545898438,
      "rewards/margins": 0.7282050848007202,
      "rewards/rejected": -2.2951834201812744,
      "sft_loss": 1.6732451915740967,
      "step": 4600
    },
    {
      "epoch": 2.464626191670848,
      "grad_norm": 7.851497136140735,
      "learning_rate": 9.374114382176457e-08,
      "logits/chosen": -0.20749421417713165,
      "logits/rejected": -0.039168864488601685,
      "logps/chosen": -1.5519232749938965,
      "logps/rejected": -2.2856459617614746,
      "loss": 0.4607,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5519232749938965,
      "rewards/margins": 0.7337230443954468,
      "rewards/rejected": -2.2856459617614746,
      "sft_loss": 1.6616754531860352,
      "step": 4605
    },
    {
      "epoch": 2.46730222445225,
      "grad_norm": 10.618025974762887,
      "learning_rate": 9.283524918896945e-08,
      "logits/chosen": -0.2528127431869507,
      "logits/rejected": -0.09626563638448715,
      "logps/chosen": -1.6161024570465088,
      "logps/rejected": -2.264498710632324,
      "loss": 0.4753,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.6161024570465088,
      "rewards/margins": 0.6483960747718811,
      "rewards/rejected": -2.264498710632324,
      "sft_loss": 1.6971594095230103,
      "step": 4610
    },
    {
      "epoch": 2.469978257233651,
      "grad_norm": 10.594551544596676,
      "learning_rate": 9.193330487037232e-08,
      "logits/chosen": -0.18787157535552979,
      "logits/rejected": -0.026337753981351852,
      "logps/chosen": -1.6552670001983643,
      "logps/rejected": -2.4452872276306152,
      "loss": 0.4485,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.6552670001983643,
      "rewards/margins": 0.7900202870368958,
      "rewards/rejected": -2.4452872276306152,
      "sft_loss": 1.7910692691802979,
      "step": 4615
    },
    {
      "epoch": 2.4726542900150528,
      "grad_norm": 7.410887227720804,
      "learning_rate": 9.103531961664118e-08,
      "logits/chosen": -0.20163783431053162,
      "logits/rejected": 0.02683967910706997,
      "logps/chosen": -1.464026689529419,
      "logps/rejected": -2.135082721710205,
      "loss": 0.4231,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.464026689529419,
      "rewards/margins": 0.67105633020401,
      "rewards/rejected": -2.135082721710205,
      "sft_loss": 1.6336700916290283,
      "step": 4620
    },
    {
      "epoch": 2.475330322796454,
      "grad_norm": 8.391502482850902,
      "learning_rate": 9.014130214003269e-08,
      "logits/chosen": -0.32311874628067017,
      "logits/rejected": -0.2665711045265198,
      "logps/chosen": -1.5374579429626465,
      "logps/rejected": -2.08915376663208,
      "loss": 0.4719,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5374579429626465,
      "rewards/margins": 0.5516960024833679,
      "rewards/rejected": -2.08915376663208,
      "sft_loss": 1.6142950057983398,
      "step": 4625
    },
    {
      "epoch": 2.4780063555778558,
      "grad_norm": 8.433282652426243,
      "learning_rate": 8.925126111430848e-08,
      "logits/chosen": -0.1706085354089737,
      "logits/rejected": -0.07626998424530029,
      "logps/chosen": -1.5323286056518555,
      "logps/rejected": -2.2087883949279785,
      "loss": 0.4505,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5323286056518555,
      "rewards/margins": 0.6764597296714783,
      "rewards/rejected": -2.2087883949279785,
      "sft_loss": 1.691072702407837,
      "step": 4630
    },
    {
      "epoch": 2.4806823883592575,
      "grad_norm": 10.54087091911546,
      "learning_rate": 8.83652051746504e-08,
      "logits/chosen": -0.14578501880168915,
      "logits/rejected": 0.02640867792069912,
      "logps/chosen": -1.5427324771881104,
      "logps/rejected": -2.2831692695617676,
      "loss": 0.4365,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5427324771881104,
      "rewards/margins": 0.7404367923736572,
      "rewards/rejected": -2.2831692695617676,
      "sft_loss": 1.5968652963638306,
      "step": 4635
    },
    {
      "epoch": 2.483358421140659,
      "grad_norm": 6.8818274016401695,
      "learning_rate": 8.748314291757696e-08,
      "logits/chosen": -0.16421422362327576,
      "logits/rejected": -0.039952315390110016,
      "logps/chosen": -1.5872137546539307,
      "logps/rejected": -2.258237838745117,
      "loss": 0.4139,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5872137546539307,
      "rewards/margins": 0.6710240244865417,
      "rewards/rejected": -2.258237838745117,
      "sft_loss": 1.6829907894134521,
      "step": 4640
    },
    {
      "epoch": 2.4860344539220605,
      "grad_norm": 7.498430334973799,
      "learning_rate": 8.660508290086032e-08,
      "logits/chosen": -0.20430341362953186,
      "logits/rejected": -0.04681653156876564,
      "logps/chosen": -1.5307071208953857,
      "logps/rejected": -2.1223111152648926,
      "loss": 0.4426,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5307071208953857,
      "rewards/margins": 0.5916039347648621,
      "rewards/rejected": -2.1223111152648926,
      "sft_loss": 1.6639989614486694,
      "step": 4645
    },
    {
      "epoch": 2.488710486703462,
      "grad_norm": 8.142152141956702,
      "learning_rate": 8.573103364344231e-08,
      "logits/chosen": -0.2613300681114197,
      "logits/rejected": 0.011642465367913246,
      "logps/chosen": -1.5273935794830322,
      "logps/rejected": -2.2350680828094482,
      "loss": 0.4326,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5273935794830322,
      "rewards/margins": 0.7076746225357056,
      "rewards/rejected": -2.2350680828094482,
      "sft_loss": 1.5699331760406494,
      "step": 4650
    },
    {
      "epoch": 2.4913865194848634,
      "grad_norm": 11.092590122881404,
      "learning_rate": 8.486100362535292e-08,
      "logits/chosen": -0.2698579430580139,
      "logits/rejected": -0.10053672641515732,
      "logps/chosen": -1.6180152893066406,
      "logps/rejected": -2.2728400230407715,
      "loss": 0.4355,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6180152893066406,
      "rewards/margins": 0.6548250317573547,
      "rewards/rejected": -2.2728400230407715,
      "sft_loss": 1.798405408859253,
      "step": 4655
    },
    {
      "epoch": 2.494062552266265,
      "grad_norm": 10.070924877852471,
      "learning_rate": 8.399500128762693e-08,
      "logits/chosen": -0.26223617792129517,
      "logits/rejected": -0.1275007724761963,
      "logps/chosen": -1.6201118230819702,
      "logps/rejected": -2.2663655281066895,
      "loss": 0.4412,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.6201118230819702,
      "rewards/margins": 0.6462540626525879,
      "rewards/rejected": -2.2663655281066895,
      "sft_loss": 1.7024576663970947,
      "step": 4660
    },
    {
      "epoch": 2.496738585047667,
      "grad_norm": 9.141394981691903,
      "learning_rate": 8.313303503222313e-08,
      "logits/chosen": -0.21037046611309052,
      "logits/rejected": -0.12881790101528168,
      "logps/chosen": -1.6491973400115967,
      "logps/rejected": -2.169259548187256,
      "loss": 0.5177,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6491973400115967,
      "rewards/margins": 0.5200623273849487,
      "rewards/rejected": -2.169259548187256,
      "sft_loss": 1.6937086582183838,
      "step": 4665
    },
    {
      "epoch": 2.4994146178290686,
      "grad_norm": 15.363046070698873,
      "learning_rate": 8.227511322194164e-08,
      "logits/chosen": -0.24200613796710968,
      "logits/rejected": -0.09714044630527496,
      "logps/chosen": -1.562156319618225,
      "logps/rejected": -2.008108615875244,
      "loss": 0.48,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.562156319618225,
      "rewards/margins": 0.4459524154663086,
      "rewards/rejected": -2.008108615875244,
      "sft_loss": 1.6635644435882568,
      "step": 4670
    },
    {
      "epoch": 2.50209065061047,
      "grad_norm": 13.921499153222394,
      "learning_rate": 8.142124418034385e-08,
      "logits/chosen": -0.163819819688797,
      "logits/rejected": 0.010681016370654106,
      "logps/chosen": -1.5085991621017456,
      "logps/rejected": -2.1517558097839355,
      "loss": 0.4396,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5085991621017456,
      "rewards/margins": 0.6431564092636108,
      "rewards/rejected": -2.1517558097839355,
      "sft_loss": 1.5810153484344482,
      "step": 4675
    },
    {
      "epoch": 2.5047666833918716,
      "grad_norm": 13.998491599616301,
      "learning_rate": 8.057143619167073e-08,
      "logits/chosen": -0.15430161356925964,
      "logits/rejected": -0.03651545196771622,
      "logps/chosen": -1.5104106664657593,
      "logps/rejected": -2.1672616004943848,
      "loss": 0.454,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5104106664657593,
      "rewards/margins": 0.6568506956100464,
      "rewards/rejected": -2.1672616004943848,
      "sft_loss": 1.551937460899353,
      "step": 4680
    },
    {
      "epoch": 2.507442716173273,
      "grad_norm": 9.441523703284677,
      "learning_rate": 7.97256975007633e-08,
      "logits/chosen": -0.29489654302597046,
      "logits/rejected": -0.07090181112289429,
      "logps/chosen": -1.531465768814087,
      "logps/rejected": -2.1749260425567627,
      "loss": 0.4285,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.531465768814087,
      "rewards/margins": 0.6434603333473206,
      "rewards/rejected": -2.1749260425567627,
      "sft_loss": 1.6144222021102905,
      "step": 4685
    },
    {
      "epoch": 2.5101187489546746,
      "grad_norm": 8.926537151475863,
      "learning_rate": 7.888403631298186e-08,
      "logits/chosen": -0.175274059176445,
      "logits/rejected": -0.11035673320293427,
      "logps/chosen": -1.5257591009140015,
      "logps/rejected": -2.1212382316589355,
      "loss": 0.4653,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5257591009140015,
      "rewards/margins": 0.5954791307449341,
      "rewards/rejected": -2.1212382316589355,
      "sft_loss": 1.582589864730835,
      "step": 4690
    },
    {
      "epoch": 2.5127947817360763,
      "grad_norm": 6.535008558907057,
      "learning_rate": 7.804646079412719e-08,
      "logits/chosen": -0.20110678672790527,
      "logits/rejected": 0.01152988150715828,
      "logps/chosen": -1.5889029502868652,
      "logps/rejected": -2.2240920066833496,
      "loss": 0.4847,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5889029502868652,
      "rewards/margins": 0.6351887583732605,
      "rewards/rejected": -2.2240920066833496,
      "sft_loss": 1.6570899486541748,
      "step": 4695
    },
    {
      "epoch": 2.515470814517478,
      "grad_norm": 9.172661414479018,
      "learning_rate": 7.72129790703604e-08,
      "logits/chosen": -0.2919153571128845,
      "logits/rejected": -0.1516752541065216,
      "logps/chosen": -1.5378618240356445,
      "logps/rejected": -2.1191885471343994,
      "loss": 0.4651,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5378618240356445,
      "rewards/margins": 0.5813267827033997,
      "rewards/rejected": -2.1191885471343994,
      "sft_loss": 1.6722614765167236,
      "step": 4700
    },
    {
      "epoch": 2.5181468472988793,
      "grad_norm": 10.146117393815855,
      "learning_rate": 7.638359922812504e-08,
      "logits/chosen": -0.16152144968509674,
      "logits/rejected": -0.10950030386447906,
      "logps/chosen": -1.5789520740509033,
      "logps/rejected": -2.131608486175537,
      "loss": 0.484,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5789520740509033,
      "rewards/margins": 0.552656352519989,
      "rewards/rejected": -2.131608486175537,
      "sft_loss": 1.6074949502944946,
      "step": 4705
    },
    {
      "epoch": 2.520822880080281,
      "grad_norm": 9.29795942326603,
      "learning_rate": 7.555832931406774e-08,
      "logits/chosen": -0.27605026960372925,
      "logits/rejected": -0.09732412546873093,
      "logps/chosen": -1.629088044166565,
      "logps/rejected": -2.297694683074951,
      "loss": 0.4428,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.629088044166565,
      "rewards/margins": 0.668606698513031,
      "rewards/rejected": -2.297694683074951,
      "sft_loss": 1.7505499124526978,
      "step": 4710
    },
    {
      "epoch": 2.5234989128616827,
      "grad_norm": 7.771825923021724,
      "learning_rate": 7.47371773349611e-08,
      "logits/chosen": -0.2481086701154709,
      "logits/rejected": -0.19722998142242432,
      "logps/chosen": -1.6821883916854858,
      "logps/rejected": -2.349914789199829,
      "loss": 0.4477,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.6821883916854858,
      "rewards/margins": 0.6677263379096985,
      "rewards/rejected": -2.349914789199829,
      "sft_loss": 1.7744452953338623,
      "step": 4715
    },
    {
      "epoch": 2.526174945643084,
      "grad_norm": 8.11669941970569,
      "learning_rate": 7.392015125762496e-08,
      "logits/chosen": -0.2124921828508377,
      "logits/rejected": -0.09005747735500336,
      "logps/chosen": -1.4214861392974854,
      "logps/rejected": -2.079423427581787,
      "loss": 0.4092,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.4214861392974854,
      "rewards/margins": 0.6579370498657227,
      "rewards/rejected": -2.079423427581787,
      "sft_loss": 1.522826910018921,
      "step": 4720
    },
    {
      "epoch": 2.5288509784244857,
      "grad_norm": 9.06402656599577,
      "learning_rate": 7.310725900885018e-08,
      "logits/chosen": -0.2556040287017822,
      "logits/rejected": -0.1857677698135376,
      "logps/chosen": -1.5797942876815796,
      "logps/rejected": -2.15522837638855,
      "loss": 0.4773,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5797942876815796,
      "rewards/margins": 0.5754343867301941,
      "rewards/rejected": -2.15522837638855,
      "sft_loss": 1.672151803970337,
      "step": 4725
    },
    {
      "epoch": 2.5315270112058874,
      "grad_norm": 6.210909136154958,
      "learning_rate": 7.229850847532076e-08,
      "logits/chosen": -0.22033897042274475,
      "logits/rejected": -0.08500131219625473,
      "logps/chosen": -1.5027967691421509,
      "logps/rejected": -2.132045269012451,
      "loss": 0.4385,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5027967691421509,
      "rewards/margins": 0.6292486786842346,
      "rewards/rejected": -2.132045269012451,
      "sft_loss": 1.6667730808258057,
      "step": 4730
    },
    {
      "epoch": 2.5342030439872887,
      "grad_norm": 9.608845457903653,
      "learning_rate": 7.149390750353779e-08,
      "logits/chosen": -0.1915491223335266,
      "logits/rejected": -0.24349656701087952,
      "logps/chosen": -1.6221160888671875,
      "logps/rejected": -2.2130630016326904,
      "loss": 0.4397,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.6221160888671875,
      "rewards/margins": 0.5909469723701477,
      "rewards/rejected": -2.2130630016326904,
      "sft_loss": 1.698195457458496,
      "step": 4735
    },
    {
      "epoch": 2.5368790767686904,
      "grad_norm": 6.093434532048691,
      "learning_rate": 7.069346389974374e-08,
      "logits/chosen": -0.288370281457901,
      "logits/rejected": -0.13510087132453918,
      "logps/chosen": -1.646715521812439,
      "logps/rejected": -2.3017725944519043,
      "loss": 0.4489,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.646715521812439,
      "rewards/margins": 0.6550571322441101,
      "rewards/rejected": -2.3017725944519043,
      "sft_loss": 1.7606605291366577,
      "step": 4740
    },
    {
      "epoch": 2.539555109550092,
      "grad_norm": 10.204274927929957,
      "learning_rate": 6.989718542984563e-08,
      "logits/chosen": -0.22085830569267273,
      "logits/rejected": -0.16141316294670105,
      "logps/chosen": -1.7018508911132812,
      "logps/rejected": -2.2902958393096924,
      "loss": 0.4871,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.7018508911132812,
      "rewards/margins": 0.5884448289871216,
      "rewards/rejected": -2.2902958393096924,
      "sft_loss": 1.768785834312439,
      "step": 4745
    },
    {
      "epoch": 2.5422311423314934,
      "grad_norm": 5.708839072897948,
      "learning_rate": 6.9105079819341e-08,
      "logits/chosen": -0.18627746403217316,
      "logits/rejected": 0.04857007414102554,
      "logps/chosen": -1.5648773908615112,
      "logps/rejected": -2.46248197555542,
      "loss": 0.3739,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5648773908615112,
      "rewards/margins": 0.8976048231124878,
      "rewards/rejected": -2.46248197555542,
      "sft_loss": 1.6237529516220093,
      "step": 4750
    },
    {
      "epoch": 2.544907175112895,
      "grad_norm": 10.876639185853666,
      "learning_rate": 6.831715475324163e-08,
      "logits/chosen": -0.2526320815086365,
      "logits/rejected": -0.06511383503675461,
      "logps/chosen": -1.4771053791046143,
      "logps/rejected": -2.261756181716919,
      "loss": 0.429,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4771053791046143,
      "rewards/margins": 0.7846506834030151,
      "rewards/rejected": -2.261756181716919,
      "sft_loss": 1.570378065109253,
      "step": 4755
    },
    {
      "epoch": 2.547583207894297,
      "grad_norm": 6.0257970161467425,
      "learning_rate": 6.753341787600026e-08,
      "logits/chosen": -0.3068903982639313,
      "logits/rejected": -0.14332722127437592,
      "logps/chosen": -1.4803614616394043,
      "logps/rejected": -2.126192331314087,
      "loss": 0.4349,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4803614616394043,
      "rewards/margins": 0.6458306312561035,
      "rewards/rejected": -2.126192331314087,
      "sft_loss": 1.6149470806121826,
      "step": 4760
    },
    {
      "epoch": 2.5502592406756985,
      "grad_norm": 8.333009520000408,
      "learning_rate": 6.67538767914353e-08,
      "logits/chosen": -0.28441572189331055,
      "logits/rejected": -0.09357120841741562,
      "logps/chosen": -1.544640064239502,
      "logps/rejected": -2.028838634490967,
      "loss": 0.5053,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.544640064239502,
      "rewards/margins": 0.484198659658432,
      "rewards/rejected": -2.028838634490967,
      "sft_loss": 1.6012932062149048,
      "step": 4765
    },
    {
      "epoch": 2.5529352734571,
      "grad_norm": 10.247917839602298,
      "learning_rate": 6.597853906265793e-08,
      "logits/chosen": -0.24177256226539612,
      "logits/rejected": -0.11481799930334091,
      "logps/chosen": -1.6507999897003174,
      "logps/rejected": -2.315504789352417,
      "loss": 0.4618,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.6507999897003174,
      "rewards/margins": 0.6647049188613892,
      "rewards/rejected": -2.315504789352417,
      "sft_loss": 1.7305183410644531,
      "step": 4770
    },
    {
      "epoch": 2.5556113062385015,
      "grad_norm": 8.516493509100233,
      "learning_rate": 6.5207412211998e-08,
      "logits/chosen": -0.11893805116415024,
      "logits/rejected": 0.0038652196526527405,
      "logps/chosen": -1.5405712127685547,
      "logps/rejected": -2.211561918258667,
      "loss": 0.4628,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5405712127685547,
      "rewards/margins": 0.6709908246994019,
      "rewards/rejected": -2.211561918258667,
      "sft_loss": 1.5511837005615234,
      "step": 4775
    },
    {
      "epoch": 2.558287339019903,
      "grad_norm": 8.246441974893472,
      "learning_rate": 6.444050372093186e-08,
      "logits/chosen": -0.21815164387226105,
      "logits/rejected": -0.10540630668401718,
      "logps/chosen": -1.6007814407348633,
      "logps/rejected": -2.1758387088775635,
      "loss": 0.4594,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6007814407348633,
      "rewards/margins": 0.5750571489334106,
      "rewards/rejected": -2.1758387088775635,
      "sft_loss": 1.6869404315948486,
      "step": 4780
    },
    {
      "epoch": 2.5609633718013045,
      "grad_norm": 7.714815198032542,
      "learning_rate": 6.367782103000873e-08,
      "logits/chosen": -0.22387388348579407,
      "logits/rejected": -0.15563735365867615,
      "logps/chosen": -1.524994134902954,
      "logps/rejected": -2.0110983848571777,
      "loss": 0.4758,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.524994134902954,
      "rewards/margins": 0.48610442876815796,
      "rewards/rejected": -2.0110983848571777,
      "sft_loss": 1.604852318763733,
      "step": 4785
    },
    {
      "epoch": 2.5636394045827062,
      "grad_norm": 6.979427476279799,
      "learning_rate": 6.29193715387798e-08,
      "logits/chosen": -0.314725786447525,
      "logits/rejected": -0.16046643257141113,
      "logps/chosen": -1.570236325263977,
      "logps/rejected": -2.2742481231689453,
      "loss": 0.4525,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.570236325263977,
      "rewards/margins": 0.7040120363235474,
      "rewards/rejected": -2.2742481231689453,
      "sft_loss": 1.6277891397476196,
      "step": 4790
    },
    {
      "epoch": 2.566315437364108,
      "grad_norm": 14.03904377279677,
      "learning_rate": 6.216516260572502e-08,
      "logits/chosen": -0.20811104774475098,
      "logits/rejected": -0.11981604248285294,
      "logps/chosen": -1.591090440750122,
      "logps/rejected": -2.183060646057129,
      "loss": 0.4741,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.591090440750122,
      "rewards/margins": 0.5919705033302307,
      "rewards/rejected": -2.183060646057129,
      "sft_loss": 1.6530237197875977,
      "step": 4795
    },
    {
      "epoch": 2.568991470145509,
      "grad_norm": 6.205669366490727,
      "learning_rate": 6.141520154818297e-08,
      "logits/chosen": -0.23087351024150848,
      "logits/rejected": -0.10391799360513687,
      "logps/chosen": -1.468090295791626,
      "logps/rejected": -2.0137531757354736,
      "loss": 0.4422,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.468090295791626,
      "rewards/margins": 0.5456628799438477,
      "rewards/rejected": -2.0137531757354736,
      "sft_loss": 1.6055347919464111,
      "step": 4800
    },
    {
      "epoch": 2.568991470145509,
      "eval_logits/chosen": 0.13403891026973724,
      "eval_logits/rejected": 0.23705637454986572,
      "eval_logps/chosen": -1.615133285522461,
      "eval_logps/rejected": -2.1880459785461426,
      "eval_loss": 0.489407479763031,
      "eval_rewards/accuracies": 0.669881284236908,
      "eval_rewards/chosen": -1.615133285522461,
      "eval_rewards/margins": 0.5729125738143921,
      "eval_rewards/rejected": -2.1880459785461426,
      "eval_runtime": 43.3666,
      "eval_samples_per_second": 31.015,
      "eval_sft_loss": 1.6890332698822021,
      "eval_steps_per_second": 7.771,
      "step": 4800
    },
    {
      "epoch": 2.571667502926911,
      "grad_norm": 9.126873683344243,
      "learning_rate": 6.066949564227897e-08,
      "logits/chosen": -0.31909486651420593,
      "logits/rejected": -0.18140074610710144,
      "logps/chosen": -1.5817447900772095,
      "logps/rejected": -2.2661678791046143,
      "loss": 0.4507,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5817447900772095,
      "rewards/margins": 0.6844232678413391,
      "rewards/rejected": -2.2661678791046143,
      "sft_loss": 1.6768858432769775,
      "step": 4805
    },
    {
      "epoch": 2.574343535708312,
      "grad_norm": 9.655498565639416,
      "learning_rate": 5.992805212285523e-08,
      "logits/chosen": -0.21249334514141083,
      "logits/rejected": -0.06652887165546417,
      "logps/chosen": -1.644213318824768,
      "logps/rejected": -2.251002788543701,
      "loss": 0.4839,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.644213318824768,
      "rewards/margins": 0.6067894697189331,
      "rewards/rejected": -2.251002788543701,
      "sft_loss": 1.711158037185669,
      "step": 4810
    },
    {
      "epoch": 2.577019568489714,
      "grad_norm": 7.29758415690169,
      "learning_rate": 5.9190878183399684e-08,
      "logits/chosen": -0.19294914603233337,
      "logits/rejected": -0.05259813740849495,
      "logps/chosen": -1.3905773162841797,
      "logps/rejected": -2.1160082817077637,
      "loss": 0.4306,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3905773162841797,
      "rewards/margins": 0.725430965423584,
      "rewards/rejected": -2.1160082817077637,
      "sft_loss": 1.5208027362823486,
      "step": 4815
    },
    {
      "epoch": 2.5796956012711156,
      "grad_norm": 8.18138114361707,
      "learning_rate": 5.845798097597748e-08,
      "logits/chosen": -0.16653354465961456,
      "logits/rejected": -0.0694139152765274,
      "logps/chosen": -1.596074104309082,
      "logps/rejected": -2.121253252029419,
      "loss": 0.4828,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.596074104309082,
      "rewards/margins": 0.5251791477203369,
      "rewards/rejected": -2.121253252029419,
      "sft_loss": 1.616479516029358,
      "step": 4820
    },
    {
      "epoch": 2.5823716340525174,
      "grad_norm": 12.856515905683183,
      "learning_rate": 5.772936761116026e-08,
      "logits/chosen": -0.19333486258983612,
      "logits/rejected": -0.05315401405096054,
      "logps/chosen": -1.5848066806793213,
      "logps/rejected": -2.146439790725708,
      "loss": 0.4625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5848066806793213,
      "rewards/margins": 0.5616329908370972,
      "rewards/rejected": -2.146439790725708,
      "sft_loss": 1.6240943670272827,
      "step": 4825
    },
    {
      "epoch": 2.5850476668339186,
      "grad_norm": 12.89972314598527,
      "learning_rate": 5.700504515795829e-08,
      "logits/chosen": -0.2448471337556839,
      "logits/rejected": -0.07540064305067062,
      "logps/chosen": -1.600976586341858,
      "logps/rejected": -2.113248109817505,
      "loss": 0.4935,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.600976586341858,
      "rewards/margins": 0.5122717022895813,
      "rewards/rejected": -2.113248109817505,
      "sft_loss": 1.6421327590942383,
      "step": 4830
    },
    {
      "epoch": 2.5877236996153203,
      "grad_norm": 8.822778276424396,
      "learning_rate": 5.628502064375101e-08,
      "logits/chosen": -0.35598570108413696,
      "logits/rejected": -0.14402401447296143,
      "logps/chosen": -1.546924352645874,
      "logps/rejected": -2.2639241218566895,
      "loss": 0.4309,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.546924352645874,
      "rewards/margins": 0.7169994711875916,
      "rewards/rejected": -2.2639241218566895,
      "sft_loss": 1.6327308416366577,
      "step": 4835
    },
    {
      "epoch": 2.5903997323967216,
      "grad_norm": 9.760793670851609,
      "learning_rate": 5.55693010542197e-08,
      "logits/chosen": -0.2984338700771332,
      "logits/rejected": -0.0784219279885292,
      "logps/chosen": -1.5313817262649536,
      "logps/rejected": -2.211430072784424,
      "loss": 0.4308,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5313817262649536,
      "rewards/margins": 0.6800485849380493,
      "rewards/rejected": -2.211430072784424,
      "sft_loss": 1.6355266571044922,
      "step": 4840
    },
    {
      "epoch": 2.5930757651781233,
      "grad_norm": 7.753373059889907,
      "learning_rate": 5.485789333327856e-08,
      "logits/chosen": -0.24122004210948944,
      "logits/rejected": -0.13646744191646576,
      "logps/chosen": -1.5357145071029663,
      "logps/rejected": -2.123054265975952,
      "loss": 0.4625,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5357145071029663,
      "rewards/margins": 0.5873396992683411,
      "rewards/rejected": -2.123054265975952,
      "sft_loss": 1.6655117273330688,
      "step": 4845
    },
    {
      "epoch": 2.595751797959525,
      "grad_norm": 10.372174907711964,
      "learning_rate": 5.4150804383008675e-08,
      "logits/chosen": -0.38891011476516724,
      "logits/rejected": -0.22265692055225372,
      "logps/chosen": -1.5839711427688599,
      "logps/rejected": -2.3328089714050293,
      "loss": 0.4537,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5839711427688599,
      "rewards/margins": 0.748837947845459,
      "rewards/rejected": -2.3328089714050293,
      "sft_loss": 1.6295592784881592,
      "step": 4850
    },
    {
      "epoch": 2.5984278307409268,
      "grad_norm": 9.388446175901082,
      "learning_rate": 5.344804106359002e-08,
      "logits/chosen": -0.21261592209339142,
      "logits/rejected": -0.046322330832481384,
      "logps/chosen": -1.467795729637146,
      "logps/rejected": -2.14953351020813,
      "loss": 0.4437,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.467795729637146,
      "rewards/margins": 0.6817376017570496,
      "rewards/rejected": -2.14953351020813,
      "sft_loss": 1.5824216604232788,
      "step": 4855
    },
    {
      "epoch": 2.601103863522328,
      "grad_norm": 9.932866439440142,
      "learning_rate": 5.274961019323559e-08,
      "logits/chosen": -0.21794529259204865,
      "logits/rejected": -0.13863089680671692,
      "logps/chosen": -1.454467535018921,
      "logps/rejected": -2.1822597980499268,
      "loss": 0.4014,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.454467535018921,
      "rewards/margins": 0.7277923822402954,
      "rewards/rejected": -2.1822597980499268,
      "sft_loss": 1.5677722692489624,
      "step": 4860
    },
    {
      "epoch": 2.6037798963037297,
      "grad_norm": 5.935104918288228,
      "learning_rate": 5.205551854812451e-08,
      "logits/chosen": -0.3364775478839874,
      "logits/rejected": -0.2209477722644806,
      "logps/chosen": -1.5508983135223389,
      "logps/rejected": -2.298210620880127,
      "loss": 0.4229,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.5508983135223389,
      "rewards/margins": 0.7473124265670776,
      "rewards/rejected": -2.298210620880127,
      "sft_loss": 1.656023621559143,
      "step": 4865
    },
    {
      "epoch": 2.606455929085131,
      "grad_norm": 8.690393763781572,
      "learning_rate": 5.1365772862337177e-08,
      "logits/chosen": -0.191738560795784,
      "logits/rejected": -0.04482860490679741,
      "logps/chosen": -1.5055210590362549,
      "logps/rejected": -2.2219274044036865,
      "loss": 0.4312,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5055210590362549,
      "rewards/margins": 0.7164064645767212,
      "rewards/rejected": -2.2219274044036865,
      "sft_loss": 1.5471540689468384,
      "step": 4870
    },
    {
      "epoch": 2.6091319618665327,
      "grad_norm": 8.615821035292372,
      "learning_rate": 5.068037982778905e-08,
      "logits/chosen": -0.04971639811992645,
      "logits/rejected": 0.04083705693483353,
      "logps/chosen": -1.4413139820098877,
      "logps/rejected": -2.260993480682373,
      "loss": 0.4066,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4413139820098877,
      "rewards/margins": 0.8196796178817749,
      "rewards/rejected": -2.260993480682373,
      "sft_loss": 1.5663115978240967,
      "step": 4875
    },
    {
      "epoch": 2.6118079946479344,
      "grad_norm": 9.24558337178255,
      "learning_rate": 4.999934609416656e-08,
      "logits/chosen": -0.15009254217147827,
      "logits/rejected": -0.01788342371582985,
      "logps/chosen": -1.460079550743103,
      "logps/rejected": -2.1505239009857178,
      "loss": 0.452,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.460079550743103,
      "rewards/margins": 0.6904444694519043,
      "rewards/rejected": -2.1505239009857178,
      "sft_loss": 1.5730839967727661,
      "step": 4880
    },
    {
      "epoch": 2.614484027429336,
      "grad_norm": 11.280343455224969,
      "learning_rate": 4.932267826886183e-08,
      "logits/chosen": -0.13067641854286194,
      "logits/rejected": -0.04214060679078102,
      "logps/chosen": -1.6614328622817993,
      "logps/rejected": -2.3834664821624756,
      "loss": 0.4509,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6614328622817993,
      "rewards/margins": 0.7220335602760315,
      "rewards/rejected": -2.3834664821624756,
      "sft_loss": 1.8111798763275146,
      "step": 4885
    },
    {
      "epoch": 2.6171600602107374,
      "grad_norm": 10.464309113105843,
      "learning_rate": 4.8650382916909206e-08,
      "logits/chosen": -0.30967697501182556,
      "logits/rejected": -0.1269526481628418,
      "logps/chosen": -1.6145133972167969,
      "logps/rejected": -2.3082594871520996,
      "loss": 0.4437,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6145133972167969,
      "rewards/margins": 0.6937457919120789,
      "rewards/rejected": -2.3082594871520996,
      "sft_loss": 1.753544807434082,
      "step": 4890
    },
    {
      "epoch": 2.619836092992139,
      "grad_norm": 6.449572000966775,
      "learning_rate": 4.7982466560920976e-08,
      "logits/chosen": -0.23348002135753632,
      "logits/rejected": -0.13083966076374054,
      "logps/chosen": -1.6567020416259766,
      "logps/rejected": -2.081791400909424,
      "loss": 0.5265,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6567020416259766,
      "rewards/margins": 0.42508941888809204,
      "rewards/rejected": -2.081791400909424,
      "sft_loss": 1.7725639343261719,
      "step": 4895
    },
    {
      "epoch": 2.622512125773541,
      "grad_norm": 7.184053345840208,
      "learning_rate": 4.7318935681024685e-08,
      "logits/chosen": -0.16638872027397156,
      "logits/rejected": -0.01653314381837845,
      "logps/chosen": -1.5108344554901123,
      "logps/rejected": -2.2297990322113037,
      "loss": 0.4326,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5108344554901123,
      "rewards/margins": 0.7189643383026123,
      "rewards/rejected": -2.2297990322113037,
      "sft_loss": 1.647355318069458,
      "step": 4900
    },
    {
      "epoch": 2.625188158554942,
      "grad_norm": 8.670758590717718,
      "learning_rate": 4.6659796714799745e-08,
      "logits/chosen": -0.22268524765968323,
      "logits/rejected": -0.03691389039158821,
      "logps/chosen": -1.6005432605743408,
      "logps/rejected": -2.360244035720825,
      "loss": 0.4281,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.6005432605743408,
      "rewards/margins": 0.7597008943557739,
      "rewards/rejected": -2.360244035720825,
      "sft_loss": 1.785644292831421,
      "step": 4905
    },
    {
      "epoch": 2.627864191336344,
      "grad_norm": 8.260339247335292,
      "learning_rate": 4.60050560572155e-08,
      "logits/chosen": -0.2165859490633011,
      "logits/rejected": -0.23379795253276825,
      "logps/chosen": -1.51551353931427,
      "logps/rejected": -2.400438070297241,
      "loss": 0.4252,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.51551353931427,
      "rewards/margins": 0.884924590587616,
      "rewards/rejected": -2.400438070297241,
      "sft_loss": 1.5888447761535645,
      "step": 4910
    },
    {
      "epoch": 2.6305402241177456,
      "grad_norm": 10.01943548791071,
      "learning_rate": 4.535472006056834e-08,
      "logits/chosen": -0.16946204006671906,
      "logits/rejected": -0.03227314352989197,
      "logps/chosen": -1.4113956689834595,
      "logps/rejected": -2.093937635421753,
      "loss": 0.4104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4113956689834595,
      "rewards/margins": 0.6825419664382935,
      "rewards/rejected": -2.093937635421753,
      "sft_loss": 1.5719985961914062,
      "step": 4915
    },
    {
      "epoch": 2.6332162568991473,
      "grad_norm": 7.831384054268323,
      "learning_rate": 4.470879503442132e-08,
      "logits/chosen": -0.178746297955513,
      "logits/rejected": -0.07838728278875351,
      "logps/chosen": -1.5404300689697266,
      "logps/rejected": -2.182508945465088,
      "loss": 0.461,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5404300689697266,
      "rewards/margins": 0.642078697681427,
      "rewards/rejected": -2.182508945465088,
      "sft_loss": 1.6610815525054932,
      "step": 4920
    },
    {
      "epoch": 2.6358922896805486,
      "grad_norm": 8.281073335530952,
      "learning_rate": 4.406728724554154e-08,
      "logits/chosen": -0.39260661602020264,
      "logits/rejected": -0.13857212662696838,
      "logps/chosen": -1.5768979787826538,
      "logps/rejected": -2.304960250854492,
      "loss": 0.4179,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5768979787826538,
      "rewards/margins": 0.7280624508857727,
      "rewards/rejected": -2.304960250854492,
      "sft_loss": 1.7252795696258545,
      "step": 4925
    },
    {
      "epoch": 2.6385683224619503,
      "grad_norm": 9.723847297600088,
      "learning_rate": 4.3430202917840664e-08,
      "logits/chosen": -0.1615389585494995,
      "logits/rejected": 0.019265184178948402,
      "logps/chosen": -1.606909990310669,
      "logps/rejected": -2.394131898880005,
      "loss": 0.4336,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.606909990310669,
      "rewards/margins": 0.7872219681739807,
      "rewards/rejected": -2.394131898880005,
      "sft_loss": 1.6091573238372803,
      "step": 4930
    },
    {
      "epoch": 2.6412443552433515,
      "grad_norm": 9.830988256385965,
      "learning_rate": 4.279754823231346e-08,
      "logits/chosen": -0.25972622632980347,
      "logits/rejected": -0.08407093584537506,
      "logps/chosen": -1.5240100622177124,
      "logps/rejected": -2.0847277641296387,
      "loss": 0.485,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5240100622177124,
      "rewards/margins": 0.5607175230979919,
      "rewards/rejected": -2.0847277641296387,
      "sft_loss": 1.6416919231414795,
      "step": 4935
    },
    {
      "epoch": 2.6439203880247533,
      "grad_norm": 6.381974017322839,
      "learning_rate": 4.216932932697859e-08,
      "logits/chosen": -0.20109012722969055,
      "logits/rejected": -0.09972207248210907,
      "logps/chosen": -1.6247422695159912,
      "logps/rejected": -2.0559937953948975,
      "loss": 0.4804,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.6247422695159912,
      "rewards/margins": 0.43125152587890625,
      "rewards/rejected": -2.0559937953948975,
      "sft_loss": 1.7206628322601318,
      "step": 4940
    },
    {
      "epoch": 2.646596420806155,
      "grad_norm": 9.754375916741807,
      "learning_rate": 4.154555229681844e-08,
      "logits/chosen": -0.24837973713874817,
      "logits/rejected": -0.027765735983848572,
      "logps/chosen": -1.5064687728881836,
      "logps/rejected": -2.308729887008667,
      "loss": 0.4307,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5064687728881836,
      "rewards/margins": 0.802261471748352,
      "rewards/rejected": -2.308729887008667,
      "sft_loss": 1.6191641092300415,
      "step": 4945
    },
    {
      "epoch": 2.6492724535875567,
      "grad_norm": 6.164449381414046,
      "learning_rate": 4.092622319372069e-08,
      "logits/chosen": -0.24197252094745636,
      "logits/rejected": -0.0956760048866272,
      "logps/chosen": -1.5101855993270874,
      "logps/rejected": -2.1189210414886475,
      "loss": 0.4513,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5101855993270874,
      "rewards/margins": 0.6087356805801392,
      "rewards/rejected": -2.1189210414886475,
      "sft_loss": 1.5725116729736328,
      "step": 4950
    },
    {
      "epoch": 2.651948486368958,
      "grad_norm": 8.143842354509493,
      "learning_rate": 4.031134802641889e-08,
      "logits/chosen": -0.19666540622711182,
      "logits/rejected": -0.19075897336006165,
      "logps/chosen": -1.4913088083267212,
      "logps/rejected": -2.094479560852051,
      "loss": 0.4552,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4913088083267212,
      "rewards/margins": 0.60317063331604,
      "rewards/rejected": -2.094479560852051,
      "sft_loss": 1.59578275680542,
      "step": 4955
    },
    {
      "epoch": 2.6546245191503597,
      "grad_norm": 11.325967769963212,
      "learning_rate": 3.970093276043468e-08,
      "logits/chosen": -0.1340693086385727,
      "logits/rejected": -0.018059438094496727,
      "logps/chosen": -1.5090904235839844,
      "logps/rejected": -2.183967113494873,
      "loss": 0.4342,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5090904235839844,
      "rewards/margins": 0.6748765110969543,
      "rewards/rejected": -2.183967113494873,
      "sft_loss": 1.553903341293335,
      "step": 4960
    },
    {
      "epoch": 2.657300551931761,
      "grad_norm": 7.159332913727375,
      "learning_rate": 3.9094983318019584e-08,
      "logits/chosen": -0.24333210289478302,
      "logits/rejected": -0.10442526638507843,
      "logps/chosen": -1.4497852325439453,
      "logps/rejected": -2.13478422164917,
      "loss": 0.4216,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4497852325439453,
      "rewards/margins": 0.6849991083145142,
      "rewards/rejected": -2.13478422164917,
      "sft_loss": 1.6094410419464111,
      "step": 4965
    },
    {
      "epoch": 2.6599765847131627,
      "grad_norm": 11.093852678407805,
      "learning_rate": 3.849350557809789e-08,
      "logits/chosen": -0.15132607519626617,
      "logits/rejected": -0.07192808389663696,
      "logps/chosen": -1.4410126209259033,
      "logps/rejected": -2.1370604038238525,
      "loss": 0.4089,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4410126209259033,
      "rewards/margins": 0.6960476636886597,
      "rewards/rejected": -2.1370604038238525,
      "sft_loss": 1.4817826747894287,
      "step": 4970
    },
    {
      "epoch": 2.6626526174945644,
      "grad_norm": 8.217590422474169,
      "learning_rate": 3.789650537620903e-08,
      "logits/chosen": -0.1982906460762024,
      "logits/rejected": -0.13623209297657013,
      "logps/chosen": -1.5887467861175537,
      "logps/rejected": -2.1640536785125732,
      "loss": 0.4649,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5887467861175537,
      "rewards/margins": 0.5753068923950195,
      "rewards/rejected": -2.1640536785125732,
      "sft_loss": 1.6395225524902344,
      "step": 4975
    },
    {
      "epoch": 2.665328650275966,
      "grad_norm": 7.250556524321402,
      "learning_rate": 3.730398850445182e-08,
      "logits/chosen": -0.14488236606121063,
      "logits/rejected": -0.067586749792099,
      "logps/chosen": -1.7181322574615479,
      "logps/rejected": -2.3087222576141357,
      "loss": 0.517,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.7181322574615479,
      "rewards/margins": 0.5905901193618774,
      "rewards/rejected": -2.3087222576141357,
      "sft_loss": 1.6792157888412476,
      "step": 4980
    },
    {
      "epoch": 2.6680046830573674,
      "grad_norm": 9.52426655213416,
      "learning_rate": 3.671596071142735e-08,
      "logits/chosen": -0.15554898977279663,
      "logits/rejected": 0.04365387558937073,
      "logps/chosen": -1.4417258501052856,
      "logps/rejected": -2.2459707260131836,
      "loss": 0.4376,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4417258501052856,
      "rewards/margins": 0.8042449951171875,
      "rewards/rejected": -2.2459707260131836,
      "sft_loss": 1.4861071109771729,
      "step": 4985
    },
    {
      "epoch": 2.670680715838769,
      "grad_norm": 11.527604544943122,
      "learning_rate": 3.6132427702183996e-08,
      "logits/chosen": -0.3058409094810486,
      "logits/rejected": -0.07154044508934021,
      "logps/chosen": -1.518927812576294,
      "logps/rejected": -2.209139108657837,
      "loss": 0.4262,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.518927812576294,
      "rewards/margins": 0.690211296081543,
      "rewards/rejected": -2.209139108657837,
      "sft_loss": 1.616014838218689,
      "step": 4990
    },
    {
      "epoch": 2.6733567486201704,
      "grad_norm": 7.802385369560936,
      "learning_rate": 3.555339513816147e-08,
      "logits/chosen": -0.24710774421691895,
      "logits/rejected": -0.2252301722764969,
      "logps/chosen": -1.5771944522857666,
      "logps/rejected": -2.0971343517303467,
      "loss": 0.466,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5771944522857666,
      "rewards/margins": 0.5199397802352905,
      "rewards/rejected": -2.0971343517303467,
      "sft_loss": 1.6544469594955444,
      "step": 4995
    },
    {
      "epoch": 2.676032781401572,
      "grad_norm": 6.263767299351617,
      "learning_rate": 3.497886863713639e-08,
      "logits/chosen": -0.22472020983695984,
      "logits/rejected": -0.20360127091407776,
      "logps/chosen": -1.6000200510025024,
      "logps/rejected": -2.1722025871276855,
      "loss": 0.4853,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.6000200510025024,
      "rewards/margins": 0.5721826553344727,
      "rewards/rejected": -2.1722025871276855,
      "sft_loss": 1.6936304569244385,
      "step": 5000
    },
    {
      "epoch": 2.678708814182974,
      "grad_norm": 7.991459840747118,
      "learning_rate": 3.440885377316721e-08,
      "logits/chosen": -0.17534606158733368,
      "logits/rejected": -0.12202417850494385,
      "logps/chosen": -1.537259817123413,
      "logps/rejected": -2.1599433422088623,
      "loss": 0.4254,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.537259817123413,
      "rewards/margins": 0.6226836442947388,
      "rewards/rejected": -2.1599433422088623,
      "sft_loss": 1.598702073097229,
      "step": 5005
    },
    {
      "epoch": 2.6813848469643755,
      "grad_norm": 8.166723627192075,
      "learning_rate": 3.384335607654082e-08,
      "logits/chosen": -0.14732176065444946,
      "logits/rejected": -0.03624642640352249,
      "logps/chosen": -1.6838470697402954,
      "logps/rejected": -2.3584134578704834,
      "loss": 0.4864,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.6838470697402954,
      "rewards/margins": 0.6745663285255432,
      "rewards/rejected": -2.3584134578704834,
      "sft_loss": 1.7807865142822266,
      "step": 5010
    },
    {
      "epoch": 2.684060879745777,
      "grad_norm": 10.626354095511608,
      "learning_rate": 3.328238103371811e-08,
      "logits/chosen": -0.25213223695755005,
      "logits/rejected": -0.1784714162349701,
      "logps/chosen": -1.581032395362854,
      "logps/rejected": -2.1678004264831543,
      "loss": 0.4511,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.581032395362854,
      "rewards/margins": 0.5867680311203003,
      "rewards/rejected": -2.1678004264831543,
      "sft_loss": 1.6110156774520874,
      "step": 5015
    },
    {
      "epoch": 2.6867369125271785,
      "grad_norm": 9.717407654604258,
      "learning_rate": 3.272593408728169e-08,
      "logits/chosen": -0.26242008805274963,
      "logits/rejected": -0.044322580099105835,
      "logps/chosen": -1.4938757419586182,
      "logps/rejected": -2.065871000289917,
      "loss": 0.4531,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4938757419586182,
      "rewards/margins": 0.5719951391220093,
      "rewards/rejected": -2.065871000289917,
      "sft_loss": 1.5849002599716187,
      "step": 5020
    },
    {
      "epoch": 2.6894129453085798,
      "grad_norm": 8.53375616791577,
      "learning_rate": 3.217402063588204e-08,
      "logits/chosen": -0.25997504591941833,
      "logits/rejected": -0.10667125880718231,
      "logps/chosen": -1.5702990293502808,
      "logps/rejected": -2.188900947570801,
      "loss": 0.4692,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5702990293502808,
      "rewards/margins": 0.6186016798019409,
      "rewards/rejected": -2.188900947570801,
      "sft_loss": 1.6220414638519287,
      "step": 5025
    },
    {
      "epoch": 2.6920889780899815,
      "grad_norm": 8.298187880845026,
      "learning_rate": 3.162664603418608e-08,
      "logits/chosen": -0.19612005352973938,
      "logits/rejected": -0.11876839399337769,
      "logps/chosen": -1.5246912240982056,
      "logps/rejected": -2.186675548553467,
      "loss": 0.474,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5246912240982056,
      "rewards/margins": 0.6619843244552612,
      "rewards/rejected": -2.186675548553467,
      "sft_loss": 1.5625849962234497,
      "step": 5030
    },
    {
      "epoch": 2.694765010871383,
      "grad_norm": 17.42258631649615,
      "learning_rate": 3.1083815592824416e-08,
      "logits/chosen": -0.2558310627937317,
      "logits/rejected": -0.12091462314128876,
      "logps/chosen": -1.6476072072982788,
      "logps/rejected": -2.227116346359253,
      "loss": 0.4678,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.6476072072982788,
      "rewards/margins": 0.5795091390609741,
      "rewards/rejected": -2.227116346359253,
      "sft_loss": 1.7288262844085693,
      "step": 5035
    },
    {
      "epoch": 2.697441043652785,
      "grad_norm": 9.098056259420295,
      "learning_rate": 3.054553457834053e-08,
      "logits/chosen": -0.035765133798122406,
      "logits/rejected": -0.07453829795122147,
      "logps/chosen": -1.5224034786224365,
      "logps/rejected": -2.1308224201202393,
      "loss": 0.4588,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5224034786224365,
      "rewards/margins": 0.6084188222885132,
      "rewards/rejected": -2.1308224201202393,
      "sft_loss": 1.5826609134674072,
      "step": 5040
    },
    {
      "epoch": 2.700117076434186,
      "grad_norm": 14.179261364612977,
      "learning_rate": 3.0011808213139036e-08,
      "logits/chosen": -0.1643834412097931,
      "logits/rejected": -0.1282930076122284,
      "logps/chosen": -1.4879213571548462,
      "logps/rejected": -2.031446933746338,
      "loss": 0.4554,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4879213571548462,
      "rewards/margins": 0.5435255169868469,
      "rewards/rejected": -2.031446933746338,
      "sft_loss": 1.5665534734725952,
      "step": 5045
    },
    {
      "epoch": 2.702793109215588,
      "grad_norm": 7.54749645668722,
      "learning_rate": 2.948264167543568e-08,
      "logits/chosen": -0.23299995064735413,
      "logits/rejected": -0.15355482697486877,
      "logps/chosen": -1.3718222379684448,
      "logps/rejected": -1.998819351196289,
      "loss": 0.4129,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.3718222379684448,
      "rewards/margins": 0.6269969344139099,
      "rewards/rejected": -1.998819351196289,
      "sft_loss": 1.4779876470565796,
      "step": 5050
    },
    {
      "epoch": 2.7054691419969896,
      "grad_norm": 9.293253168136241,
      "learning_rate": 2.8958040099206216e-08,
      "logits/chosen": -0.3311172425746918,
      "logits/rejected": -0.24270012974739075,
      "logps/chosen": -1.3977210521697998,
      "logps/rejected": -2.0432395935058594,
      "loss": 0.4268,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3977210521697998,
      "rewards/margins": 0.6455187797546387,
      "rewards/rejected": -2.0432395935058594,
      "sft_loss": 1.497981071472168,
      "step": 5055
    },
    {
      "epoch": 2.708145174778391,
      "grad_norm": 11.595698661147347,
      "learning_rate": 2.843800857413775e-08,
      "logits/chosen": -0.16359257698059082,
      "logits/rejected": -0.09543822705745697,
      "logps/chosen": -1.5558207035064697,
      "logps/rejected": -2.118010997772217,
      "loss": 0.4694,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5558207035064697,
      "rewards/margins": 0.5621901750564575,
      "rewards/rejected": -2.118010997772217,
      "sft_loss": 1.6672084331512451,
      "step": 5060
    },
    {
      "epoch": 2.7108212075597926,
      "grad_norm": 14.84432234803869,
      "learning_rate": 2.7922552145578203e-08,
      "logits/chosen": -0.26126575469970703,
      "logits/rejected": -0.00785075407475233,
      "logps/chosen": -1.5473171472549438,
      "logps/rejected": -2.1697630882263184,
      "loss": 0.4402,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5473171472549438,
      "rewards/margins": 0.6224457025527954,
      "rewards/rejected": -2.1697630882263184,
      "sft_loss": 1.6175849437713623,
      "step": 5065
    },
    {
      "epoch": 2.7134972403411943,
      "grad_norm": 7.8924081270315725,
      "learning_rate": 2.7411675814488277e-08,
      "logits/chosen": -0.10214686393737793,
      "logits/rejected": 0.0706239640712738,
      "logps/chosen": -1.4367588758468628,
      "logps/rejected": -1.9674361944198608,
      "loss": 0.4426,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4367588758468628,
      "rewards/margins": 0.5306774377822876,
      "rewards/rejected": -1.9674361944198608,
      "sft_loss": 1.5512819290161133,
      "step": 5070
    },
    {
      "epoch": 2.7161732731225956,
      "grad_norm": 13.96875352505859,
      "learning_rate": 2.690538453739216e-08,
      "logits/chosen": -0.21306836605072021,
      "logits/rejected": -0.12876783311367035,
      "logps/chosen": -1.5248085260391235,
      "logps/rejected": -1.9554111957550049,
      "loss": 0.502,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.5248085260391235,
      "rewards/margins": 0.4306026101112366,
      "rewards/rejected": -1.9554111957550049,
      "sft_loss": 1.6193593740463257,
      "step": 5075
    },
    {
      "epoch": 2.7188493059039973,
      "grad_norm": 7.563130631641448,
      "learning_rate": 2.6403683226330298e-08,
      "logits/chosen": -0.2365640103816986,
      "logits/rejected": -0.08295810222625732,
      "logps/chosen": -1.5342161655426025,
      "logps/rejected": -2.123009204864502,
      "loss": 0.4521,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5342161655426025,
      "rewards/margins": 0.5887928605079651,
      "rewards/rejected": -2.123009204864502,
      "sft_loss": 1.6141750812530518,
      "step": 5080
    },
    {
      "epoch": 2.721525338685399,
      "grad_norm": 14.560564045470185,
      "learning_rate": 2.5906576748810804e-08,
      "logits/chosen": -0.2672147750854492,
      "logits/rejected": -0.13339033722877502,
      "logps/chosen": -1.4195791482925415,
      "logps/rejected": -2.135160446166992,
      "loss": 0.4187,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.4195791482925415,
      "rewards/margins": 0.7155812978744507,
      "rewards/rejected": -2.135160446166992,
      "sft_loss": 1.4889144897460938,
      "step": 5085
    },
    {
      "epoch": 2.7242013714668003,
      "grad_norm": 11.143544002117181,
      "learning_rate": 2.5414069927763016e-08,
      "logits/chosen": -0.34407171607017517,
      "logits/rejected": -0.16657240688800812,
      "logps/chosen": -1.5777204036712646,
      "logps/rejected": -2.2593626976013184,
      "loss": 0.4461,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5777204036712646,
      "rewards/margins": 0.6816423535346985,
      "rewards/rejected": -2.2593626976013184,
      "sft_loss": 1.6737632751464844,
      "step": 5090
    },
    {
      "epoch": 2.726877404248202,
      "grad_norm": 9.455706044788442,
      "learning_rate": 2.4926167541490185e-08,
      "logits/chosen": -0.35349464416503906,
      "logits/rejected": -0.1343749463558197,
      "logps/chosen": -1.5024068355560303,
      "logps/rejected": -2.2888574600219727,
      "loss": 0.4242,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5024068355560303,
      "rewards/margins": 0.7864503860473633,
      "rewards/rejected": -2.2888574600219727,
      "sft_loss": 1.5991122722625732,
      "step": 5095
    },
    {
      "epoch": 2.7295534370296037,
      "grad_norm": 7.712027144423693,
      "learning_rate": 2.4442874323623574e-08,
      "logits/chosen": -0.13933254778385162,
      "logits/rejected": 0.02814735472202301,
      "logps/chosen": -1.4643703699111938,
      "logps/rejected": -2.3610987663269043,
      "loss": 0.4017,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4643703699111938,
      "rewards/margins": 0.8967283964157104,
      "rewards/rejected": -2.3610987663269043,
      "sft_loss": 1.5425755977630615,
      "step": 5100
    },
    {
      "epoch": 2.7322294698110055,
      "grad_norm": 11.281307044328882,
      "learning_rate": 2.396419496307589e-08,
      "logits/chosen": -0.20330731570720673,
      "logits/rejected": -0.023081596940755844,
      "logps/chosen": -1.5841712951660156,
      "logps/rejected": -2.14113712310791,
      "loss": 0.4599,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5841712951660156,
      "rewards/margins": 0.5569659471511841,
      "rewards/rejected": -2.14113712310791,
      "sft_loss": 1.6328054666519165,
      "step": 5105
    },
    {
      "epoch": 2.7349055025924067,
      "grad_norm": 9.18953732538059,
      "learning_rate": 2.349013410399653e-08,
      "logits/chosen": -0.28744104504585266,
      "logits/rejected": -0.13670162856578827,
      "logps/chosen": -1.5169239044189453,
      "logps/rejected": -2.094639539718628,
      "loss": 0.4631,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5169239044189453,
      "rewards/margins": 0.5777156352996826,
      "rewards/rejected": -2.094639539718628,
      "sft_loss": 1.5577342510223389,
      "step": 5110
    },
    {
      "epoch": 2.7375815353738084,
      "grad_norm": 6.65679864485864,
      "learning_rate": 2.3020696345725954e-08,
      "logits/chosen": -0.30195218324661255,
      "logits/rejected": -0.06154744699597359,
      "logps/chosen": -1.5235546827316284,
      "logps/rejected": -2.314807415008545,
      "loss": 0.4186,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5235546827316284,
      "rewards/margins": 0.7912526726722717,
      "rewards/rejected": -2.314807415008545,
      "sft_loss": 1.599188208580017,
      "step": 5115
    },
    {
      "epoch": 2.7402575681552097,
      "grad_norm": 9.923834819757321,
      "learning_rate": 2.2555886242751398e-08,
      "logits/chosen": -0.24068517982959747,
      "logits/rejected": -0.1709970235824585,
      "logps/chosen": -1.641798734664917,
      "logps/rejected": -2.3314099311828613,
      "loss": 0.4287,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.641798734664917,
      "rewards/margins": 0.6896110773086548,
      "rewards/rejected": -2.3314099311828613,
      "sft_loss": 1.6708080768585205,
      "step": 5120
    },
    {
      "epoch": 2.7429336009366114,
      "grad_norm": 23.02297917390171,
      "learning_rate": 2.2095708304662453e-08,
      "logits/chosen": -0.35668811202049255,
      "logits/rejected": -0.1095414012670517,
      "logps/chosen": -1.5242853164672852,
      "logps/rejected": -2.1377031803131104,
      "loss": 0.445,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5242853164672852,
      "rewards/margins": 0.6134177446365356,
      "rewards/rejected": -2.1377031803131104,
      "sft_loss": 1.6786664724349976,
      "step": 5125
    },
    {
      "epoch": 2.745609633718013,
      "grad_norm": 6.036728353405439,
      "learning_rate": 2.16401669961076e-08,
      "logits/chosen": -0.3969551920890808,
      "logits/rejected": -0.189706951379776,
      "logps/chosen": -1.5305182933807373,
      "logps/rejected": -2.223459482192993,
      "loss": 0.4451,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5305182933807373,
      "rewards/margins": 0.6929410099983215,
      "rewards/rejected": -2.223459482192993,
      "sft_loss": 1.66098952293396,
      "step": 5130
    },
    {
      "epoch": 2.748285666499415,
      "grad_norm": 10.095901836204533,
      "learning_rate": 2.1189266736750532e-08,
      "logits/chosen": -0.16776272654533386,
      "logits/rejected": -0.08403909206390381,
      "logps/chosen": -1.5309851169586182,
      "logps/rejected": -2.127441883087158,
      "loss": 0.4487,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5309851169586182,
      "rewards/margins": 0.5964566469192505,
      "rewards/rejected": -2.127441883087158,
      "sft_loss": 1.6376146078109741,
      "step": 5135
    },
    {
      "epoch": 2.750961699280816,
      "grad_norm": 8.905118245131481,
      "learning_rate": 2.0743011901227623e-08,
      "logits/chosen": -0.18821024894714355,
      "logits/rejected": -0.03925692290067673,
      "logps/chosen": -1.6175647974014282,
      "logps/rejected": -2.1396234035491943,
      "loss": 0.4854,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.6175647974014282,
      "rewards/margins": 0.52205890417099,
      "rewards/rejected": -2.1396234035491943,
      "sft_loss": 1.666550636291504,
      "step": 5140
    },
    {
      "epoch": 2.753637732062218,
      "grad_norm": 8.405618016929523,
      "learning_rate": 2.030140681910508e-08,
      "logits/chosen": -0.258244127035141,
      "logits/rejected": -0.07392361015081406,
      "logps/chosen": -1.4821580648422241,
      "logps/rejected": -2.103391170501709,
      "loss": 0.4643,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4821580648422241,
      "rewards/margins": 0.6212331056594849,
      "rewards/rejected": -2.103391170501709,
      "sft_loss": 1.6087650060653687,
      "step": 5145
    },
    {
      "epoch": 2.756313764843619,
      "grad_norm": 5.673376367465625,
      "learning_rate": 1.986445577483753e-08,
      "logits/chosen": -0.3048526346683502,
      "logits/rejected": -0.1681317389011383,
      "logps/chosen": -1.4941984415054321,
      "logps/rejected": -2.1286990642547607,
      "loss": 0.42,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4941984415054321,
      "rewards/margins": 0.6345007419586182,
      "rewards/rejected": -2.1286990642547607,
      "sft_loss": 1.573027491569519,
      "step": 5150
    },
    {
      "epoch": 2.758989797625021,
      "grad_norm": 8.00899310290481,
      "learning_rate": 1.9432163007725765e-08,
      "logits/chosen": -0.3116549849510193,
      "logits/rejected": -0.19156138598918915,
      "logps/chosen": -1.551615834236145,
      "logps/rejected": -2.102240800857544,
      "loss": 0.4607,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.551615834236145,
      "rewards/margins": 0.5506250858306885,
      "rewards/rejected": -2.102240800857544,
      "sft_loss": 1.6373153924942017,
      "step": 5155
    },
    {
      "epoch": 2.7616658304064226,
      "grad_norm": 5.466003891775965,
      "learning_rate": 1.9004532711876297e-08,
      "logits/chosen": -0.2839297652244568,
      "logits/rejected": -0.21033921837806702,
      "logps/chosen": -1.5315322875976562,
      "logps/rejected": -2.277865409851074,
      "loss": 0.4046,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.5315322875976562,
      "rewards/margins": 0.7463330030441284,
      "rewards/rejected": -2.277865409851074,
      "sft_loss": 1.680361032485962,
      "step": 5160
    },
    {
      "epoch": 2.7643418631878243,
      "grad_norm": 10.590259452652054,
      "learning_rate": 1.8581569036159928e-08,
      "logits/chosen": -0.23004582524299622,
      "logits/rejected": -0.027688121423125267,
      "logps/chosen": -1.4334943294525146,
      "logps/rejected": -2.082756757736206,
      "loss": 0.4271,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4334943294525146,
      "rewards/margins": 0.6492623686790466,
      "rewards/rejected": -2.082756757736206,
      "sft_loss": 1.511309027671814,
      "step": 5165
    },
    {
      "epoch": 2.7670178959692255,
      "grad_norm": 6.041999725545914,
      "learning_rate": 1.8163276084172285e-08,
      "logits/chosen": -0.2488495111465454,
      "logits/rejected": -0.10540600121021271,
      "logps/chosen": -1.5621020793914795,
      "logps/rejected": -2.1619415283203125,
      "loss": 0.4774,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5621020793914795,
      "rewards/margins": 0.5998395085334778,
      "rewards/rejected": -2.1619415283203125,
      "sft_loss": 1.6926653385162354,
      "step": 5170
    },
    {
      "epoch": 2.7696939287506273,
      "grad_norm": 7.86482733723809,
      "learning_rate": 1.7749657914193194e-08,
      "logits/chosen": -0.266471266746521,
      "logits/rejected": -0.14478352665901184,
      "logps/chosen": -1.5819045305252075,
      "logps/rejected": -2.25170636177063,
      "loss": 0.4435,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5819045305252075,
      "rewards/margins": 0.6698015332221985,
      "rewards/rejected": -2.25170636177063,
      "sft_loss": 1.6461341381072998,
      "step": 5175
    },
    {
      "epoch": 2.7723699615320285,
      "grad_norm": 8.961045368620963,
      "learning_rate": 1.7340718539148203e-08,
      "logits/chosen": -0.17507778108119965,
      "logits/rejected": -0.10742446035146713,
      "logps/chosen": -1.6066890954971313,
      "logps/rejected": -2.258471965789795,
      "loss": 0.4773,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6066890954971313,
      "rewards/margins": 0.6517831087112427,
      "rewards/rejected": -2.258471965789795,
      "sft_loss": 1.7125810384750366,
      "step": 5180
    },
    {
      "epoch": 2.7750459943134302,
      "grad_norm": 7.230794407833966,
      "learning_rate": 1.6936461926568724e-08,
      "logits/chosen": -0.1980207860469818,
      "logits/rejected": -0.0723222941160202,
      "logps/chosen": -1.4438279867172241,
      "logps/rejected": -2.1929969787597656,
      "loss": 0.4185,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4438279867172241,
      "rewards/margins": 0.7491689324378967,
      "rewards/rejected": -2.1929969787597656,
      "sft_loss": 1.5771210193634033,
      "step": 5185
    },
    {
      "epoch": 2.777722027094832,
      "grad_norm": 8.572718412601594,
      "learning_rate": 1.6536891998554346e-08,
      "logits/chosen": -0.3288986384868622,
      "logits/rejected": -0.14068400859832764,
      "logps/chosen": -1.4717166423797607,
      "logps/rejected": -2.157830238342285,
      "loss": 0.4282,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4717166423797607,
      "rewards/margins": 0.686113715171814,
      "rewards/rejected": -2.157830238342285,
      "sft_loss": 1.6161715984344482,
      "step": 5190
    },
    {
      "epoch": 2.7803980598762337,
      "grad_norm": 10.395910525815902,
      "learning_rate": 1.6142012631734093e-08,
      "logits/chosen": -0.2193346917629242,
      "logits/rejected": -0.07468325644731522,
      "logps/chosen": -1.5009483098983765,
      "logps/rejected": -2.134226083755493,
      "loss": 0.4188,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5009483098983765,
      "rewards/margins": 0.633277952671051,
      "rewards/rejected": -2.134226083755493,
      "sft_loss": 1.554665446281433,
      "step": 5195
    },
    {
      "epoch": 2.783074092657635,
      "grad_norm": 11.82919190666528,
      "learning_rate": 1.575182765722949e-08,
      "logits/chosen": -0.31428205966949463,
      "logits/rejected": -0.13497485220432281,
      "logps/chosen": -1.517218828201294,
      "logps/rejected": -2.152031898498535,
      "loss": 0.4463,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.517218828201294,
      "rewards/margins": 0.634813129901886,
      "rewards/rejected": -2.152031898498535,
      "sft_loss": 1.6881401538848877,
      "step": 5200
    },
    {
      "epoch": 2.783074092657635,
      "eval_logits/chosen": 0.13055913150310516,
      "eval_logits/rejected": 0.23271715641021729,
      "eval_logps/chosen": -1.6105517148971558,
      "eval_logps/rejected": -2.1856274604797363,
      "eval_loss": 0.48953157663345337,
      "eval_rewards/accuracies": 0.6706231236457825,
      "eval_rewards/chosen": -1.6105517148971558,
      "eval_rewards/margins": 0.5750758647918701,
      "eval_rewards/rejected": -2.1856274604797363,
      "eval_runtime": 43.0982,
      "eval_samples_per_second": 31.208,
      "eval_sft_loss": 1.6850907802581787,
      "eval_steps_per_second": 7.819,
      "step": 5200
    },
    {
      "epoch": 2.7857501254390367,
      "grad_norm": 5.776006694489402,
      "learning_rate": 1.536634086061672e-08,
      "logits/chosen": -0.16184869408607483,
      "logits/rejected": -0.11565609276294708,
      "logps/chosen": -1.522731065750122,
      "logps/rejected": -2.048495292663574,
      "loss": 0.4919,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.522731065750122,
      "rewards/margins": 0.525763988494873,
      "rewards/rejected": -2.048495292663574,
      "sft_loss": 1.5864933729171753,
      "step": 5205
    },
    {
      "epoch": 2.788426158220438,
      "grad_norm": 11.013142341793925,
      "learning_rate": 1.4985555981890495e-08,
      "logits/chosen": -0.23087449371814728,
      "logits/rejected": -0.12524442374706268,
      "logps/chosen": -1.5012990236282349,
      "logps/rejected": -2.2111098766326904,
      "loss": 0.4237,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5012990236282349,
      "rewards/margins": 0.7098108530044556,
      "rewards/rejected": -2.2111098766326904,
      "sft_loss": 1.5755040645599365,
      "step": 5210
    },
    {
      "epoch": 2.7911021910018396,
      "grad_norm": 6.685725664190028,
      "learning_rate": 1.4609476715427226e-08,
      "logits/chosen": -0.2300528585910797,
      "logits/rejected": -0.11930929124355316,
      "logps/chosen": -1.4321932792663574,
      "logps/rejected": -2.1179568767547607,
      "loss": 0.4317,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4321932792663574,
      "rewards/margins": 0.6857637166976929,
      "rewards/rejected": -2.1179568767547607,
      "sft_loss": 1.5366168022155762,
      "step": 5215
    },
    {
      "epoch": 2.7937782237832414,
      "grad_norm": 10.542810332554014,
      "learning_rate": 1.4238106709949792e-08,
      "logits/chosen": -0.26986294984817505,
      "logits/rejected": -0.18295058608055115,
      "logps/chosen": -1.4609721899032593,
      "logps/rejected": -2.1687216758728027,
      "loss": 0.4393,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4609721899032593,
      "rewards/margins": 0.7077494263648987,
      "rewards/rejected": -2.1687216758728027,
      "sft_loss": 1.616141676902771,
      "step": 5220
    },
    {
      "epoch": 2.796454256564643,
      "grad_norm": 11.314222661472881,
      "learning_rate": 1.3871449568491511e-08,
      "logits/chosen": -0.17271623015403748,
      "logits/rejected": -0.020607611164450645,
      "logps/chosen": -1.5896047353744507,
      "logps/rejected": -2.263044834136963,
      "loss": 0.4608,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5896047353744507,
      "rewards/margins": 0.6734402179718018,
      "rewards/rejected": -2.263044834136963,
      "sft_loss": 1.6785653829574585,
      "step": 5225
    },
    {
      "epoch": 2.7991302893460444,
      "grad_norm": 8.33161220717835,
      "learning_rate": 1.3509508848361606e-08,
      "logits/chosen": -0.3504981994628906,
      "logits/rejected": -0.19691799581050873,
      "logps/chosen": -1.5263431072235107,
      "logps/rejected": -2.1571273803710938,
      "loss": 0.4404,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5263431072235107,
      "rewards/margins": 0.6307845711708069,
      "rewards/rejected": -2.1571273803710938,
      "sft_loss": 1.58456289768219,
      "step": 5230
    },
    {
      "epoch": 2.801806322127446,
      "grad_norm": 8.253808280426956,
      "learning_rate": 1.3152288061110517e-08,
      "logits/chosen": -0.294038861989975,
      "logits/rejected": -0.15794646739959717,
      "logps/chosen": -1.5321094989776611,
      "logps/rejected": -2.1294748783111572,
      "loss": 0.4538,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5321094989776611,
      "rewards/margins": 0.5973653793334961,
      "rewards/rejected": -2.1294748783111572,
      "sft_loss": 1.5454280376434326,
      "step": 5235
    },
    {
      "epoch": 2.804482354908848,
      "grad_norm": 8.614401323815699,
      "learning_rate": 1.2799790672495814e-08,
      "logits/chosen": -0.28644105792045593,
      "logits/rejected": -0.0708000659942627,
      "logps/chosen": -1.5604805946350098,
      "logps/rejected": -2.216197967529297,
      "loss": 0.4628,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5604805946350098,
      "rewards/margins": 0.6557173132896423,
      "rewards/rejected": -2.216197967529297,
      "sft_loss": 1.628729224205017,
      "step": 5240
    },
    {
      "epoch": 2.807158387690249,
      "grad_norm": 8.606419640888854,
      "learning_rate": 1.2452020102448835e-08,
      "logits/chosen": -0.16988658905029297,
      "logits/rejected": -0.11387205123901367,
      "logps/chosen": -1.4814785718917847,
      "logps/rejected": -2.048768997192383,
      "loss": 0.4741,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4814785718917847,
      "rewards/margins": 0.5672904253005981,
      "rewards/rejected": -2.048768997192383,
      "sft_loss": 1.5562591552734375,
      "step": 5245
    },
    {
      "epoch": 2.8098344204716508,
      "grad_norm": 12.649362573939653,
      "learning_rate": 1.2108979725041103e-08,
      "logits/chosen": -0.3165106773376465,
      "logits/rejected": -0.15842103958129883,
      "logps/chosen": -1.6255061626434326,
      "logps/rejected": -2.3039708137512207,
      "loss": 0.4743,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.6255061626434326,
      "rewards/margins": 0.6784645915031433,
      "rewards/rejected": -2.3039708137512207,
      "sft_loss": 1.7424743175506592,
      "step": 5250
    },
    {
      "epoch": 2.8125104532530525,
      "grad_norm": 12.29929108395042,
      "learning_rate": 1.1770672868451958e-08,
      "logits/chosen": -0.2662120759487152,
      "logits/rejected": -0.02815375290811062,
      "logps/chosen": -1.5816794633865356,
      "logps/rejected": -2.2056572437286377,
      "loss": 0.4591,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5816794633865356,
      "rewards/margins": 0.6239776015281677,
      "rewards/rejected": -2.2056572437286377,
      "sft_loss": 1.6130762100219727,
      "step": 5255
    },
    {
      "epoch": 2.8151864860344538,
      "grad_norm": 8.28428810284338,
      "learning_rate": 1.1437102814935872e-08,
      "logits/chosen": -0.22748716175556183,
      "logits/rejected": -0.1518431007862091,
      "logps/chosen": -1.5503032207489014,
      "logps/rejected": -2.2076878547668457,
      "loss": 0.4621,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.5503032207489014,
      "rewards/margins": 0.6573843359947205,
      "rewards/rejected": -2.2076878547668457,
      "sft_loss": 1.7244062423706055,
      "step": 5260
    },
    {
      "epoch": 2.8178625188158555,
      "grad_norm": 8.376139146060785,
      "learning_rate": 1.1108272800791018e-08,
      "logits/chosen": -0.38391146063804626,
      "logits/rejected": -0.13936465978622437,
      "logps/chosen": -1.784136414527893,
      "logps/rejected": -2.388622760772705,
      "loss": 0.4983,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.784136414527893,
      "rewards/margins": 0.604486346244812,
      "rewards/rejected": -2.388622760772705,
      "sft_loss": 1.860600471496582,
      "step": 5265
    },
    {
      "epoch": 2.820538551597257,
      "grad_norm": 7.011574259425963,
      "learning_rate": 1.078418601632769e-08,
      "logits/chosen": -0.23148080706596375,
      "logits/rejected": -0.07034826278686523,
      "logps/chosen": -1.5404936075210571,
      "logps/rejected": -2.229719638824463,
      "loss": 0.4208,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5404936075210571,
      "rewards/margins": 0.6892261505126953,
      "rewards/rejected": -2.229719638824463,
      "sft_loss": 1.607832670211792,
      "step": 5270
    },
    {
      "epoch": 2.8232145843786585,
      "grad_norm": 7.240883226597204,
      "learning_rate": 1.0464845605837159e-08,
      "logits/chosen": -0.2358875274658203,
      "logits/rejected": -0.07125989347696304,
      "logps/chosen": -1.6322225332260132,
      "logps/rejected": -2.355457067489624,
      "loss": 0.4138,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6322225332260132,
      "rewards/margins": 0.7232344746589661,
      "rewards/rejected": -2.355457067489624,
      "sft_loss": 1.6577503681182861,
      "step": 5275
    },
    {
      "epoch": 2.82589061716006,
      "grad_norm": 9.271934661143398,
      "learning_rate": 1.0150254667561642e-08,
      "logits/chosen": -0.24580411612987518,
      "logits/rejected": -0.0542788989841938,
      "logps/chosen": -1.715682029724121,
      "logps/rejected": -2.4755477905273438,
      "loss": 0.4507,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.715682029724121,
      "rewards/margins": 0.7598655819892883,
      "rewards/rejected": -2.4755477905273438,
      "sft_loss": 1.7498623132705688,
      "step": 5280
    },
    {
      "epoch": 2.828566649941462,
      "grad_norm": 11.374744424713278,
      "learning_rate": 9.840416253663719e-09,
      "logits/chosen": -0.30271807312965393,
      "logits/rejected": -0.1887938678264618,
      "logps/chosen": -1.4635353088378906,
      "logps/rejected": -2.2433948516845703,
      "loss": 0.4178,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4635353088378906,
      "rewards/margins": 0.7798596620559692,
      "rewards/rejected": -2.2433948516845703,
      "sft_loss": 1.5233333110809326,
      "step": 5285
    },
    {
      "epoch": 2.8312426827228636,
      "grad_norm": 6.872953005493221,
      "learning_rate": 9.535333370197074e-09,
      "logits/chosen": -0.2544511556625366,
      "logits/rejected": -0.096409872174263,
      "logps/chosen": -1.5046459436416626,
      "logps/rejected": -2.1032772064208984,
      "loss": 0.4494,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5046459436416626,
      "rewards/margins": 0.5986312031745911,
      "rewards/rejected": -2.1032772064208984,
      "sft_loss": 1.6099576950073242,
      "step": 5290
    },
    {
      "epoch": 2.833918715504265,
      "grad_norm": 6.137494826088779,
      "learning_rate": 9.23500897707713e-09,
      "logits/chosen": -0.3068126440048218,
      "logits/rejected": -0.09982781112194061,
      "logps/chosen": -1.6437880992889404,
      "logps/rejected": -2.3714306354522705,
      "loss": 0.4506,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.6437880992889404,
      "rewards/margins": 0.7276423573493958,
      "rewards/rejected": -2.3714306354522705,
      "sft_loss": 1.72063410282135,
      "step": 5295
    },
    {
      "epoch": 2.8365947482856666,
      "grad_norm": 9.158112466999524,
      "learning_rate": 8.939445988052574e-09,
      "logits/chosen": -0.26331356167793274,
      "logits/rejected": -0.21247005462646484,
      "logps/chosen": -1.5546457767486572,
      "logps/rejected": -2.288158893585205,
      "loss": 0.4343,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5546457767486572,
      "rewards/margins": 0.7335132360458374,
      "rewards/rejected": -2.288158893585205,
      "sft_loss": 1.5880502462387085,
      "step": 5300
    },
    {
      "epoch": 2.839270781067068,
      "grad_norm": 8.260041502015017,
      "learning_rate": 8.648647270676656e-09,
      "logits/chosen": -0.2249482423067093,
      "logits/rejected": -0.07068298757076263,
      "logps/chosen": -1.59334397315979,
      "logps/rejected": -2.2703146934509277,
      "loss": 0.4335,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.59334397315979,
      "rewards/margins": 0.6769706606864929,
      "rewards/rejected": -2.2703146934509277,
      "sft_loss": 1.7753381729125977,
      "step": 5305
    },
    {
      "epoch": 2.8419468138484696,
      "grad_norm": 8.102204753805093,
      "learning_rate": 8.362615646279991e-09,
      "logits/chosen": -0.3869312107563019,
      "logits/rejected": -0.12273693084716797,
      "logps/chosen": -1.4944441318511963,
      "logps/rejected": -2.322719097137451,
      "loss": 0.4284,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4944441318511963,
      "rewards/margins": 0.8282749056816101,
      "rewards/rejected": -2.322719097137451,
      "sft_loss": 1.5989134311676025,
      "step": 5310
    },
    {
      "epoch": 2.8446228466298713,
      "grad_norm": 10.578547394286801,
      "learning_rate": 8.081353889942466e-09,
      "logits/chosen": -0.13628318905830383,
      "logits/rejected": 0.0010598197113722563,
      "logps/chosen": -1.5226942300796509,
      "logps/rejected": -2.0845484733581543,
      "loss": 0.4386,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5226942300796509,
      "rewards/margins": 0.5618543028831482,
      "rewards/rejected": -2.0845484733581543,
      "sft_loss": 1.6184759140014648,
      "step": 5315
    },
    {
      "epoch": 2.847298879411273,
      "grad_norm": 6.059569829129419,
      "learning_rate": 7.804864730467042e-09,
      "logits/chosen": -0.15193864703178406,
      "logits/rejected": -0.059926360845565796,
      "logps/chosen": -1.4939788579940796,
      "logps/rejected": -2.059293270111084,
      "loss": 0.4537,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4939788579940796,
      "rewards/margins": 0.565314531326294,
      "rewards/rejected": -2.059293270111084,
      "sft_loss": 1.5195624828338623,
      "step": 5320
    },
    {
      "epoch": 2.8499749121926743,
      "grad_norm": 6.775886783493703,
      "learning_rate": 7.533150850352665e-09,
      "logits/chosen": -0.21538333594799042,
      "logits/rejected": -0.05382348224520683,
      "logps/chosen": -1.5950740575790405,
      "logps/rejected": -2.3566675186157227,
      "loss": 0.4172,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.5950740575790405,
      "rewards/margins": 0.7615932822227478,
      "rewards/rejected": -2.3566675186157227,
      "sft_loss": 1.656071424484253,
      "step": 5325
    },
    {
      "epoch": 2.852650944974076,
      "grad_norm": 10.365899347234771,
      "learning_rate": 7.2662148857686175e-09,
      "logits/chosen": -0.15810880064964294,
      "logits/rejected": -0.06887652724981308,
      "logps/chosen": -1.4541465044021606,
      "logps/rejected": -2.1913342475891113,
      "loss": 0.4422,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4541465044021606,
      "rewards/margins": 0.7371879816055298,
      "rewards/rejected": -2.1913342475891113,
      "sft_loss": 1.5949485301971436,
      "step": 5330
    },
    {
      "epoch": 2.8553269777554773,
      "grad_norm": 11.959122542324632,
      "learning_rate": 7.0040594265287635e-09,
      "logits/chosen": -0.10423209518194199,
      "logits/rejected": -0.12887991964817047,
      "logps/chosen": -1.5037696361541748,
      "logps/rejected": -1.9653781652450562,
      "loss": 0.5121,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5037696361541748,
      "rewards/margins": 0.46160855889320374,
      "rewards/rejected": -1.9653781652450562,
      "sft_loss": 1.541277527809143,
      "step": 5335
    },
    {
      "epoch": 2.858003010536879,
      "grad_norm": 7.756715618841046,
      "learning_rate": 6.746687016066566e-09,
      "logits/chosen": -0.1794317662715912,
      "logits/rejected": -0.12930874526500702,
      "logps/chosen": -1.5389997959136963,
      "logps/rejected": -2.155897855758667,
      "loss": 0.4506,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5389997959136963,
      "rewards/margins": 0.6168978810310364,
      "rewards/rejected": -2.155897855758667,
      "sft_loss": 1.5519967079162598,
      "step": 5340
    },
    {
      "epoch": 2.8606790433182807,
      "grad_norm": 5.211057613264154,
      "learning_rate": 6.494100151410276e-09,
      "logits/chosen": -0.3666417896747589,
      "logits/rejected": -0.17199210822582245,
      "logps/chosen": -1.5276671648025513,
      "logps/rejected": -2.213324546813965,
      "loss": 0.401,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.5276671648025513,
      "rewards/margins": 0.6856572031974792,
      "rewards/rejected": -2.213324546813965,
      "sft_loss": 1.60750412940979,
      "step": 5345
    },
    {
      "epoch": 2.8633550760996824,
      "grad_norm": 8.446061188353697,
      "learning_rate": 6.246301283158728e-09,
      "logits/chosen": -0.15566708147525787,
      "logits/rejected": -0.1720820665359497,
      "logps/chosen": -1.6055930852890015,
      "logps/rejected": -2.133291721343994,
      "loss": 0.4838,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.6055930852890015,
      "rewards/margins": 0.5276986360549927,
      "rewards/rejected": -2.133291721343994,
      "sft_loss": 1.6225850582122803,
      "step": 5350
    },
    {
      "epoch": 2.8660311088810837,
      "grad_norm": 10.755228467455938,
      "learning_rate": 6.0032928154576944e-09,
      "logits/chosen": -0.27801212668418884,
      "logits/rejected": -0.18713174760341644,
      "logps/chosen": -1.5514525175094604,
      "logps/rejected": -2.1258859634399414,
      "loss": 0.4492,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5514525175094604,
      "rewards/margins": 0.5744332671165466,
      "rewards/rejected": -2.1258859634399414,
      "sft_loss": 1.624772071838379,
      "step": 5355
    },
    {
      "epoch": 2.8687071416624854,
      "grad_norm": 12.113195896228424,
      "learning_rate": 5.76507710597629e-09,
      "logits/chosen": -0.2438342124223709,
      "logits/rejected": -0.038455717265605927,
      "logps/chosen": -1.5836608409881592,
      "logps/rejected": -2.2291040420532227,
      "loss": 0.4598,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5836608409881592,
      "rewards/margins": 0.645443320274353,
      "rewards/rejected": -2.2291040420532227,
      "sft_loss": 1.6981513500213623,
      "step": 5360
    },
    {
      "epoch": 2.8713831744438867,
      "grad_norm": 8.03902053145534,
      "learning_rate": 5.531656465884438e-09,
      "logits/chosen": -0.3179672062397003,
      "logits/rejected": -0.13477668166160583,
      "logps/chosen": -1.579353928565979,
      "logps/rejected": -2.28670597076416,
      "loss": 0.435,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.579353928565979,
      "rewards/margins": 0.7073520421981812,
      "rewards/rejected": -2.28670597076416,
      "sft_loss": 1.6819642782211304,
      "step": 5365
    },
    {
      "epoch": 2.8740592072252884,
      "grad_norm": 8.56323599419119,
      "learning_rate": 5.303033159830217e-09,
      "logits/chosen": -0.1567922681570053,
      "logits/rejected": -0.10974302142858505,
      "logps/chosen": -1.5311553478240967,
      "logps/rejected": -1.984129548072815,
      "loss": 0.4859,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5311553478240967,
      "rewards/margins": 0.45297402143478394,
      "rewards/rejected": -1.984129548072815,
      "sft_loss": 1.6002174615859985,
      "step": 5370
    },
    {
      "epoch": 2.87673524000669,
      "grad_norm": 7.536547968436513,
      "learning_rate": 5.079209405917939e-09,
      "logits/chosen": -0.22333934903144836,
      "logits/rejected": -0.11946012079715729,
      "logps/chosen": -1.4816691875457764,
      "logps/rejected": -2.308711051940918,
      "loss": 0.4509,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4816691875457764,
      "rewards/margins": 0.8270419239997864,
      "rewards/rejected": -2.308711051940918,
      "sft_loss": 1.6006828546524048,
      "step": 5375
    },
    {
      "epoch": 2.879411272788092,
      "grad_norm": 7.189079618381418,
      "learning_rate": 4.860187375686664e-09,
      "logits/chosen": -0.2728548049926758,
      "logits/rejected": -0.02265390381217003,
      "logps/chosen": -1.6843229532241821,
      "logps/rejected": -2.4106035232543945,
      "loss": 0.427,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.6843229532241821,
      "rewards/margins": 0.7262802124023438,
      "rewards/rejected": -2.4106035232543945,
      "sft_loss": 1.7908289432525635,
      "step": 5380
    },
    {
      "epoch": 2.882087305569493,
      "grad_norm": 5.956115684859829,
      "learning_rate": 4.64596919408905e-09,
      "logits/chosen": -0.16116848587989807,
      "logits/rejected": -0.058946527540683746,
      "logps/chosen": -1.526186227798462,
      "logps/rejected": -2.0922622680664062,
      "loss": 0.4382,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.526186227798462,
      "rewards/margins": 0.5660760402679443,
      "rewards/rejected": -2.0922622680664062,
      "sft_loss": 1.606529951095581,
      "step": 5385
    },
    {
      "epoch": 2.884763338350895,
      "grad_norm": 5.904020734651611,
      "learning_rate": 4.436556939470814e-09,
      "logits/chosen": -0.2092897593975067,
      "logits/rejected": -0.06519066542387009,
      "logps/chosen": -1.5699594020843506,
      "logps/rejected": -2.089143753051758,
      "loss": 0.4774,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5699594020843506,
      "rewards/margins": 0.5191842913627625,
      "rewards/rejected": -2.089143753051758,
      "sft_loss": 1.6545159816741943,
      "step": 5390
    },
    {
      "epoch": 2.887439371132296,
      "grad_norm": 7.017257247779873,
      "learning_rate": 4.23195264355064e-09,
      "logits/chosen": -0.3744376301765442,
      "logits/rejected": -0.1616448163986206,
      "logps/chosen": -1.518035650253296,
      "logps/rejected": -2.1544997692108154,
      "loss": 0.4119,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.518035650253296,
      "rewards/margins": 0.6364642977714539,
      "rewards/rejected": -2.1544997692108154,
      "sft_loss": 1.6293920278549194,
      "step": 5395
    },
    {
      "epoch": 2.890115403913698,
      "grad_norm": 8.617520189475039,
      "learning_rate": 4.032158291400245e-09,
      "logits/chosen": -0.283908486366272,
      "logits/rejected": -0.018505841493606567,
      "logps/chosen": -1.5354034900665283,
      "logps/rejected": -2.4752886295318604,
      "loss": 0.399,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5354034900665283,
      "rewards/margins": 0.9398850202560425,
      "rewards/rejected": -2.4752886295318604,
      "sft_loss": 1.5902059078216553,
      "step": 5400
    },
    {
      "epoch": 2.8927914366950995,
      "grad_norm": 8.529954126666453,
      "learning_rate": 3.837175821425398e-09,
      "logits/chosen": -0.18799303472042084,
      "logits/rejected": -0.13106395304203033,
      "logps/chosen": -1.7391777038574219,
      "logps/rejected": -2.3081436157226562,
      "loss": 0.4983,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.7391777038574219,
      "rewards/margins": 0.5689657926559448,
      "rewards/rejected": -2.3081436157226562,
      "sft_loss": 1.776010274887085,
      "step": 5405
    },
    {
      "epoch": 2.8954674694765012,
      "grad_norm": 6.961561366310143,
      "learning_rate": 3.6470071253467683e-09,
      "logits/chosen": -0.19545699656009674,
      "logits/rejected": -0.05803150683641434,
      "logps/chosen": -1.5638599395751953,
      "logps/rejected": -2.421759605407715,
      "loss": 0.4352,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5638599395751953,
      "rewards/margins": 0.8578997850418091,
      "rewards/rejected": -2.421759605407715,
      "sft_loss": 1.6521934270858765,
      "step": 5410
    },
    {
      "epoch": 2.8981435022579025,
      "grad_norm": 6.546250379775348,
      "learning_rate": 3.461654048181939e-09,
      "logits/chosen": -0.28004032373428345,
      "logits/rejected": -0.08701594918966293,
      "logps/chosen": -1.5976712703704834,
      "logps/rejected": -2.1546006202697754,
      "loss": 0.4614,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5976712703704834,
      "rewards/margins": 0.5569295883178711,
      "rewards/rejected": -2.1546006202697754,
      "sft_loss": 1.7328250408172607,
      "step": 5415
    },
    {
      "epoch": 2.9008195350393042,
      "grad_norm": 7.927715183147347,
      "learning_rate": 3.281118388227255e-09,
      "logits/chosen": -0.19358518719673157,
      "logits/rejected": -0.11841396987438202,
      "logps/chosen": -1.5791889429092407,
      "logps/rejected": -2.156836986541748,
      "loss": 0.4691,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5791889429092407,
      "rewards/margins": 0.5776482224464417,
      "rewards/rejected": -2.156836986541748,
      "sft_loss": 1.697723388671875,
      "step": 5420
    },
    {
      "epoch": 2.903495567820706,
      "grad_norm": 9.515705477914373,
      "learning_rate": 3.1054018970405048e-09,
      "logits/chosen": -0.22499966621398926,
      "logits/rejected": -0.07803558558225632,
      "logps/chosen": -1.5573455095291138,
      "logps/rejected": -2.2448649406433105,
      "loss": 0.4359,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5573455095291138,
      "rewards/margins": 0.6875195503234863,
      "rewards/rejected": -2.2448649406433105,
      "sft_loss": 1.6308892965316772,
      "step": 5425
    },
    {
      "epoch": 2.906171600602107,
      "grad_norm": 7.606091817512331,
      "learning_rate": 2.9345062794238207e-09,
      "logits/chosen": -0.2587551772594452,
      "logits/rejected": -0.07017064839601517,
      "logps/chosen": -1.4989105463027954,
      "logps/rejected": -2.214676856994629,
      "loss": 0.4116,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4989105463027954,
      "rewards/margins": 0.7157663106918335,
      "rewards/rejected": -2.214676856994629,
      "sft_loss": 1.556244134902954,
      "step": 5430
    },
    {
      "epoch": 2.908847633383509,
      "grad_norm": 11.543911209924728,
      "learning_rate": 2.7684331934072492e-09,
      "logits/chosen": -0.3492546081542969,
      "logits/rejected": -0.2342105656862259,
      "logps/chosen": -1.4990334510803223,
      "logps/rejected": -2.2174575328826904,
      "loss": 0.4287,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4990334510803223,
      "rewards/margins": 0.7184239029884338,
      "rewards/rejected": -2.2174575328826904,
      "sft_loss": 1.606949806213379,
      "step": 5435
    },
    {
      "epoch": 2.9115236661649107,
      "grad_norm": 8.20323293946449,
      "learning_rate": 2.6071842502326526e-09,
      "logits/chosen": -0.2740515172481537,
      "logits/rejected": -0.13928236067295074,
      "logps/chosen": -1.4924064874649048,
      "logps/rejected": -2.071157217025757,
      "loss": 0.446,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4924064874649048,
      "rewards/margins": 0.5787509679794312,
      "rewards/rejected": -2.071157217025757,
      "sft_loss": 1.600365400314331,
      "step": 5440
    },
    {
      "epoch": 2.9141996989463124,
      "grad_norm": 7.721124999933021,
      "learning_rate": 2.450761014337888e-09,
      "logits/chosen": -0.03778085485100746,
      "logits/rejected": -0.022445034235715866,
      "logps/chosen": -1.5717326402664185,
      "logps/rejected": -2.443424701690674,
      "loss": 0.4363,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5717326402664185,
      "rewards/margins": 0.8716920018196106,
      "rewards/rejected": -2.443424701690674,
      "sft_loss": 1.6421794891357422,
      "step": 5445
    },
    {
      "epoch": 2.9168757317277136,
      "grad_norm": 8.705912983045286,
      "learning_rate": 2.299165003341985e-09,
      "logits/chosen": -0.14163485169410706,
      "logits/rejected": -0.04321925342082977,
      "logps/chosen": -1.607598066329956,
      "logps/rejected": -2.3242135047912598,
      "loss": 0.4599,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.607598066329956,
      "rewards/margins": 0.7166159152984619,
      "rewards/rejected": -2.3242135047912598,
      "sft_loss": 1.6503245830535889,
      "step": 5450
    },
    {
      "epoch": 2.9195517645091154,
      "grad_norm": 6.81702621455927,
      "learning_rate": 2.1523976880299945e-09,
      "logits/chosen": -0.2439926117658615,
      "logits/rejected": -0.05459132790565491,
      "logps/chosen": -1.5861202478408813,
      "logps/rejected": -2.08335280418396,
      "loss": 0.4986,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5861202478408813,
      "rewards/margins": 0.4972327649593353,
      "rewards/rejected": -2.08335280418396,
      "sft_loss": 1.6445051431655884,
      "step": 5455
    },
    {
      "epoch": 2.9222277972905166,
      "grad_norm": 8.199519234548399,
      "learning_rate": 2.010460492339161e-09,
      "logits/chosen": -0.2169719934463501,
      "logits/rejected": -0.09205500036478043,
      "logps/chosen": -1.548842430114746,
      "logps/rejected": -2.2640018463134766,
      "loss": 0.4295,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.548842430114746,
      "rewards/margins": 0.7151594758033752,
      "rewards/rejected": -2.2640018463134766,
      "sft_loss": 1.6609394550323486,
      "step": 5460
    },
    {
      "epoch": 2.9249038300719183,
      "grad_norm": 7.551700105436862,
      "learning_rate": 1.8733547933446614e-09,
      "logits/chosen": -0.3121129274368286,
      "logits/rejected": -0.1062546968460083,
      "logps/chosen": -1.6494537591934204,
      "logps/rejected": -2.213712692260742,
      "loss": 0.4601,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6494537591934204,
      "rewards/margins": 0.5642590522766113,
      "rewards/rejected": -2.213712692260742,
      "sft_loss": 1.6747217178344727,
      "step": 5465
    },
    {
      "epoch": 2.92757986285332,
      "grad_norm": 11.08626706640028,
      "learning_rate": 1.7410819212467231e-09,
      "logits/chosen": -0.1887563318014145,
      "logits/rejected": -0.09307411313056946,
      "logps/chosen": -1.5149710178375244,
      "logps/rejected": -2.1049327850341797,
      "loss": 0.4609,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5149710178375244,
      "rewards/margins": 0.5899616479873657,
      "rewards/rejected": -2.1049327850341797,
      "sft_loss": 1.585853934288025,
      "step": 5470
    },
    {
      "epoch": 2.9302558956347218,
      "grad_norm": 9.496446162853063,
      "learning_rate": 1.613643159357192e-09,
      "logits/chosen": -0.14326107501983643,
      "logits/rejected": -0.17542067170143127,
      "logps/chosen": -1.4481487274169922,
      "logps/rejected": -2.071676731109619,
      "loss": 0.4234,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4481487274169922,
      "rewards/margins": 0.6235281229019165,
      "rewards/rejected": -2.071676731109619,
      "sft_loss": 1.5847502946853638,
      "step": 5475
    },
    {
      "epoch": 2.932931928416123,
      "grad_norm": 6.652368802603641,
      "learning_rate": 1.4910397440875967e-09,
      "logits/chosen": -0.23582640290260315,
      "logits/rejected": -0.11037939786911011,
      "logps/chosen": -1.614134430885315,
      "logps/rejected": -2.2239456176757812,
      "loss": 0.4576,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.614134430885315,
      "rewards/margins": 0.6098113059997559,
      "rewards/rejected": -2.2239456176757812,
      "sft_loss": 1.7315566539764404,
      "step": 5480
    },
    {
      "epoch": 2.9356079611975248,
      "grad_norm": 6.233736541983424,
      "learning_rate": 1.3732728649368253e-09,
      "logits/chosen": -0.13952019810676575,
      "logits/rejected": 0.0490092858672142,
      "logps/chosen": -1.4858424663543701,
      "logps/rejected": -2.1162209510803223,
      "loss": 0.4321,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4858424663543701,
      "rewards/margins": 0.6303783059120178,
      "rewards/rejected": -2.1162209510803223,
      "sft_loss": 1.5842537879943848,
      "step": 5485
    },
    {
      "epoch": 2.938283993978926,
      "grad_norm": 12.749067219519082,
      "learning_rate": 1.260343664479524e-09,
      "logits/chosen": -0.22309139370918274,
      "logits/rejected": -0.1758689433336258,
      "logps/chosen": -1.4894559383392334,
      "logps/rejected": -2.0888736248016357,
      "loss": 0.4462,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4894559383392334,
      "rewards/margins": 0.5994176864624023,
      "rewards/rejected": -2.0888736248016357,
      "sft_loss": 1.6486419439315796,
      "step": 5490
    },
    {
      "epoch": 2.9409600267603278,
      "grad_norm": 11.22414600791383,
      "learning_rate": 1.1522532383554384e-09,
      "logits/chosen": -0.30279210209846497,
      "logits/rejected": -0.09396946430206299,
      "logps/chosen": -1.50836980342865,
      "logps/rejected": -2.2401037216186523,
      "loss": 0.4341,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.50836980342865,
      "rewards/margins": 0.7317339777946472,
      "rewards/rejected": -2.2401037216186523,
      "sft_loss": 1.6456451416015625,
      "step": 5495
    },
    {
      "epoch": 2.9436360595417295,
      "grad_norm": 7.056685347275519,
      "learning_rate": 1.049002635258256e-09,
      "logits/chosen": -0.21237125992774963,
      "logits/rejected": -0.08999466896057129,
      "logps/chosen": -1.5916156768798828,
      "logps/rejected": -2.1415927410125732,
      "loss": 0.4845,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5916156768798828,
      "rewards/margins": 0.54997718334198,
      "rewards/rejected": -2.1415927410125732,
      "sft_loss": 1.6495697498321533,
      "step": 5500
    },
    {
      "epoch": 2.946312092323131,
      "grad_norm": 8.831089583385387,
      "learning_rate": 9.505928569258358e-10,
      "logits/chosen": -0.15597601234912872,
      "logits/rejected": -0.14214563369750977,
      "logps/chosen": -1.5390175580978394,
      "logps/rejected": -2.1770451068878174,
      "loss": 0.4375,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.5390175580978394,
      "rewards/margins": 0.638027548789978,
      "rewards/rejected": -2.1770451068878174,
      "sft_loss": 1.6056034564971924,
      "step": 5505
    },
    {
      "epoch": 2.9489881251045325,
      "grad_norm": 11.814580230373496,
      "learning_rate": 8.57024858130273e-10,
      "logits/chosen": -0.25163984298706055,
      "logits/rejected": -0.11963468790054321,
      "logps/chosen": -1.5341298580169678,
      "logps/rejected": -2.457284688949585,
      "loss": 0.409,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.5341298580169678,
      "rewards/margins": 0.9231548309326172,
      "rewards/rejected": -2.457284688949585,
      "sft_loss": 1.578685998916626,
      "step": 5510
    },
    {
      "epoch": 2.951664157885934,
      "grad_norm": 12.25741083867556,
      "learning_rate": 7.682995466686826e-10,
      "logits/chosen": -0.3177322745323181,
      "logits/rejected": -0.16120585799217224,
      "logps/chosen": -1.503438949584961,
      "logps/rejected": -2.218707323074341,
      "loss": 0.4506,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.503438949584961,
      "rewards/margins": 0.7152682542800903,
      "rewards/rejected": -2.218707323074341,
      "sft_loss": 1.6588207483291626,
      "step": 5515
    },
    {
      "epoch": 2.9543401906673354,
      "grad_norm": 12.822732341972932,
      "learning_rate": 6.844177833543741e-10,
      "logits/chosen": -0.2096444070339203,
      "logits/rejected": -0.13994260132312775,
      "logps/chosen": -1.5128350257873535,
      "logps/rejected": -2.0648093223571777,
      "loss": 0.4593,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5128350257873535,
      "rewards/margins": 0.551974356174469,
      "rewards/rejected": -2.0648093223571777,
      "sft_loss": 1.592486023902893,
      "step": 5520
    },
    {
      "epoch": 2.957016223448737,
      "grad_norm": 10.009567514790708,
      "learning_rate": 6.053803820087467e-10,
      "logits/chosen": -0.2408403605222702,
      "logits/rejected": -0.11644059419631958,
      "logps/chosen": -1.6379438638687134,
      "logps/rejected": -2.3291876316070557,
      "loss": 0.4608,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6379438638687134,
      "rewards/margins": 0.6912436485290527,
      "rewards/rejected": -2.3291876316070557,
      "sft_loss": 1.751821517944336,
      "step": 5525
    },
    {
      "epoch": 2.959692256230139,
      "grad_norm": 10.66253837775219,
      "learning_rate": 5.311881094528514e-10,
      "logits/chosen": -0.2995717525482178,
      "logits/rejected": -0.05560945346951485,
      "logps/chosen": -1.6378438472747803,
      "logps/rejected": -2.179105043411255,
      "loss": 0.4816,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6378438472747803,
      "rewards/margins": 0.5412610769271851,
      "rewards/rejected": -2.179105043411255,
      "sft_loss": 1.6909475326538086,
      "step": 5530
    },
    {
      "epoch": 2.9623682890115406,
      "grad_norm": 8.164331998457351,
      "learning_rate": 4.6184168550050806e-10,
      "logits/chosen": -0.2505989074707031,
      "logits/rejected": -0.1925095021724701,
      "logps/chosen": -1.5192012786865234,
      "logps/rejected": -2.0587570667266846,
      "loss": 0.485,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5192012786865234,
      "rewards/margins": 0.539555549621582,
      "rewards/rejected": -2.0587570667266846,
      "sft_loss": 1.6563823223114014,
      "step": 5535
    },
    {
      "epoch": 2.965044321792942,
      "grad_norm": 7.643442799207146,
      "learning_rate": 3.973417829510328e-10,
      "logits/chosen": -0.36065754294395447,
      "logits/rejected": -0.19505465030670166,
      "logps/chosen": -1.5969563722610474,
      "logps/rejected": -2.221297025680542,
      "loss": 0.4724,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5969563722610474,
      "rewards/margins": 0.6243407130241394,
      "rewards/rejected": -2.221297025680542,
      "sft_loss": 1.6127878427505493,
      "step": 5540
    },
    {
      "epoch": 2.9677203545743436,
      "grad_norm": 15.705846713940252,
      "learning_rate": 3.3768902758274377e-10,
      "logits/chosen": -0.23518851399421692,
      "logits/rejected": -0.10746979713439941,
      "logps/chosen": -1.4499380588531494,
      "logps/rejected": -2.019010305404663,
      "loss": 0.4394,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4499380588531494,
      "rewards/margins": 0.5690725445747375,
      "rewards/rejected": -2.019010305404663,
      "sft_loss": 1.5194131135940552,
      "step": 5545
    },
    {
      "epoch": 2.970396387355745,
      "grad_norm": 10.003051386769481,
      "learning_rate": 2.8288399814691e-10,
      "logits/chosen": -0.144705131649971,
      "logits/rejected": -0.019480938091874123,
      "logps/chosen": -1.602990746498108,
      "logps/rejected": -2.247408390045166,
      "loss": 0.4483,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.602990746498108,
      "rewards/margins": 0.644417941570282,
      "rewards/rejected": -2.247408390045166,
      "sft_loss": 1.7288663387298584,
      "step": 5550
    },
    {
      "epoch": 2.9730724201371466,
      "grad_norm": 12.223390468985686,
      "learning_rate": 2.3292722636220066e-10,
      "logits/chosen": -0.24656164646148682,
      "logits/rejected": -0.019894180819392204,
      "logps/chosen": -1.6426786184310913,
      "logps/rejected": -2.375697612762451,
      "loss": 0.4522,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.6426786184310913,
      "rewards/margins": 0.7330188751220703,
      "rewards/rejected": -2.375697612762451,
      "sft_loss": 1.6906198263168335,
      "step": 5555
    },
    {
      "epoch": 2.9757484529185483,
      "grad_norm": 8.816260901663746,
      "learning_rate": 1.8781919690946668e-10,
      "logits/chosen": -0.15205910801887512,
      "logits/rejected": -0.1319962441921234,
      "logps/chosen": -1.562274694442749,
      "logps/rejected": -2.0063271522521973,
      "loss": 0.5216,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.562274694442749,
      "rewards/margins": 0.4440523684024811,
      "rewards/rejected": -2.0063271522521973,
      "sft_loss": 1.6867120265960693,
      "step": 5560
    },
    {
      "epoch": 2.97842448569995,
      "grad_norm": 7.306427594246043,
      "learning_rate": 1.4756034742696711e-10,
      "logits/chosen": -0.266589879989624,
      "logits/rejected": -0.19822144508361816,
      "logps/chosen": -1.5395902395248413,
      "logps/rejected": -2.1543092727661133,
      "loss": 0.4559,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5395902395248413,
      "rewards/margins": 0.6147189140319824,
      "rewards/rejected": -2.1543092727661133,
      "sft_loss": 1.6022942066192627,
      "step": 5565
    },
    {
      "epoch": 2.9811005184813513,
      "grad_norm": 16.13599776839836,
      "learning_rate": 1.12151068506261e-10,
      "logits/chosen": -0.19161126017570496,
      "logits/rejected": -0.04991975054144859,
      "logps/chosen": -1.4886987209320068,
      "logps/rejected": -2.400658130645752,
      "loss": 0.3969,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4886987209320068,
      "rewards/margins": 0.9119596481323242,
      "rewards/rejected": -2.400658130645752,
      "sft_loss": 1.6483510732650757,
      "step": 5570
    },
    {
      "epoch": 2.983776551262753,
      "grad_norm": 7.869560116788056,
      "learning_rate": 8.159170368826629e-11,
      "logits/chosen": -0.20632624626159668,
      "logits/rejected": -0.04783254116773605,
      "logps/chosen": -1.4557554721832275,
      "logps/rejected": -2.1471357345581055,
      "loss": 0.4296,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4557554721832275,
      "rewards/margins": 0.6913801431655884,
      "rewards/rejected": -2.1471357345581055,
      "sft_loss": 1.5793284177780151,
      "step": 5575
    },
    {
      "epoch": 2.9864525840441547,
      "grad_norm": 6.332669004351588,
      "learning_rate": 5.588254946015114e-11,
      "logits/chosen": -0.3329741656780243,
      "logits/rejected": -0.05530470609664917,
      "logps/chosen": -1.46047043800354,
      "logps/rejected": -2.1376914978027344,
      "loss": 0.4416,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.46047043800354,
      "rewards/margins": 0.6772211194038391,
      "rewards/rejected": -2.1376914978027344,
      "sft_loss": 1.5921342372894287,
      "step": 5580
    },
    {
      "epoch": 2.989128616825556,
      "grad_norm": 5.125291681931396,
      "learning_rate": 3.502385525216978e-11,
      "logits/chosen": -0.27868136763572693,
      "logits/rejected": -0.09898083657026291,
      "logps/chosen": -1.59736168384552,
      "logps/rejected": -2.370673418045044,
      "loss": 0.4277,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.59736168384552,
      "rewards/margins": 0.7733116149902344,
      "rewards/rejected": -2.370673418045044,
      "sft_loss": 1.7875791788101196,
      "step": 5585
    },
    {
      "epoch": 2.9918046496069577,
      "grad_norm": 6.56238996651261,
      "learning_rate": 1.901582343555308e-11,
      "logits/chosen": -0.19537453353405,
      "logits/rejected": -0.1279398500919342,
      "logps/chosen": -1.6379623413085938,
      "logps/rejected": -2.2381696701049805,
      "loss": 0.4926,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.6379623413085938,
      "rewards/margins": 0.6002073287963867,
      "rewards/rejected": -2.2381696701049805,
      "sft_loss": 1.667297124862671,
      "step": 5590
    },
    {
      "epoch": 2.9944806823883594,
      "grad_norm": 9.565512529644018,
      "learning_rate": 7.858609320232634e-12,
      "logits/chosen": -0.21644096076488495,
      "logits/rejected": -0.056559689342975616,
      "logps/chosen": -1.4373853206634521,
      "logps/rejected": -2.0811617374420166,
      "loss": 0.4365,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4373853206634521,
      "rewards/margins": 0.6437762975692749,
      "rewards/rejected": -2.0811617374420166,
      "sft_loss": 1.5225098133087158,
      "step": 5595
    },
    {
      "epoch": 2.9971567151697607,
      "grad_norm": 8.896316771073908,
      "learning_rate": 1.5523211535639624e-12,
      "logits/chosen": -0.21377773582935333,
      "logits/rejected": -0.0971156433224678,
      "logps/chosen": -1.5980726480484009,
      "logps/rejected": -2.448322057723999,
      "loss": 0.4311,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5980726480484009,
      "rewards/margins": 0.8502495884895325,
      "rewards/rejected": -2.448322057723999,
      "sft_loss": 1.7738758325576782,
      "step": 5600
    },
    {
      "epoch": 2.9971567151697607,
      "eval_logits/chosen": 0.1345709264278412,
      "eval_logits/rejected": 0.23717787861824036,
      "eval_logps/chosen": -1.6158757209777832,
      "eval_logps/rejected": -2.1942226886749268,
      "eval_loss": 0.4895861744880676,
      "eval_rewards/accuracies": 0.671364963054657,
      "eval_rewards/chosen": -1.6158757209777832,
      "eval_rewards/margins": 0.5783470869064331,
      "eval_rewards/rejected": -2.1942226886749268,
      "eval_runtime": 43.0679,
      "eval_samples_per_second": 31.23,
      "eval_sft_loss": 1.6913232803344727,
      "eval_steps_per_second": 7.825,
      "step": 5600
    },
    {
      "epoch": 2.999297541394882,
      "step": 5604,
      "total_flos": 0.0,
      "train_loss": 0.49661777635968474,
      "train_runtime": 31536.5493,
      "train_samples_per_second": 5.688,
      "train_steps_per_second": 0.178
    }
  ],
  "logging_steps": 5,
  "max_steps": 5604,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}