{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.5702695629416383,
  "eval_steps": 100,
  "global_step": 1500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.3333333333333334e-09,
      "logits/chosen": -2.606384754180908,
      "logits/rejected": -2.4982504844665527,
      "logps/chosen": -296.16644287109375,
      "logps/rejected": -225.2230224609375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.3333333333333334e-08,
      "logits/chosen": -2.517352819442749,
      "logits/rejected": -2.4637084007263184,
      "logps/chosen": -342.76483154296875,
      "logps/rejected": -293.0290222167969,
      "loss": 0.6947,
      "rewards/accuracies": 0.4305555522441864,
      "rewards/chosen": 0.0013976963236927986,
      "rewards/margins": -0.0040576523169875145,
      "rewards/rejected": 0.005455348175019026,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -2.396883726119995,
      "logits/rejected": -2.3743655681610107,
      "logps/chosen": -238.6938018798828,
      "logps/rejected": -248.8633575439453,
      "loss": 0.6943,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0019892356358468533,
      "rewards/margins": -0.0029293105471879244,
      "rewards/rejected": 0.0009400752605870366,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1e-07,
      "logits/chosen": -2.490886688232422,
      "logits/rejected": -2.408034563064575,
      "logps/chosen": -289.0502014160156,
      "logps/rejected": -257.93157958984375,
      "loss": 0.6915,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 8.620424341643229e-05,
      "rewards/margins": 0.0004866129602305591,
      "rewards/rejected": -0.000400408694986254,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -2.479222536087036,
      "logits/rejected": -2.4444398880004883,
      "logps/chosen": -298.08795166015625,
      "logps/rejected": -280.84173583984375,
      "loss": 0.6898,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.010273179039359093,
      "rewards/margins": 0.005869547836482525,
      "rewards/rejected": 0.004403631202876568,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -2.505727767944336,
      "logits/rejected": -2.4711194038391113,
      "logps/chosen": -268.0919494628906,
      "logps/rejected": -254.45358276367188,
      "loss": 0.6838,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.024473872035741806,
      "rewards/margins": 0.018633713945746422,
      "rewards/rejected": 0.0058401599526405334,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 2e-07,
      "logits/chosen": -2.456902265548706,
      "logits/rejected": -2.411407947540283,
      "logps/chosen": -293.38934326171875,
      "logps/rejected": -269.47882080078125,
      "loss": 0.677,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.04791393131017685,
      "rewards/margins": 0.03436511009931564,
      "rewards/rejected": 0.01354882400482893,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -2.463378429412842,
      "logits/rejected": -2.385895013809204,
      "logps/chosen": -290.9529724121094,
      "logps/rejected": -276.04107666015625,
      "loss": 0.6614,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.09689084440469742,
      "rewards/margins": 0.058380376547575,
      "rewards/rejected": 0.03851046413183212,
      "step": 70
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.6666666666666667e-07,
      "logits/chosen": -2.4076247215270996,
      "logits/rejected": -2.3244595527648926,
      "logps/chosen": -321.92596435546875,
      "logps/rejected": -284.5071105957031,
      "loss": 0.6387,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.15204539895057678,
      "rewards/margins": 0.11996610462665558,
      "rewards/rejected": 0.032079294323921204,
      "step": 80
    },
    {
      "epoch": 0.09,
      "learning_rate": 3e-07,
      "logits/chosen": -2.3717925548553467,
      "logits/rejected": -2.362457752227783,
      "logps/chosen": -314.3221435546875,
      "logps/rejected": -272.3472595214844,
      "loss": 0.6254,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.159285306930542,
      "rewards/margins": 0.18115119636058807,
      "rewards/rejected": -0.02186589315533638,
      "step": 90
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -2.3045754432678223,
      "logits/rejected": -2.326310634613037,
      "logps/chosen": -290.2933349609375,
      "logps/rejected": -319.2317199707031,
      "loss": 0.6163,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.04698774218559265,
      "rewards/margins": 0.2375941276550293,
      "rewards/rejected": -0.19060640037059784,
      "step": 100
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.3541910648345947,
      "eval_logits/rejected": -2.306934118270874,
      "eval_logps/chosen": -296.0254211425781,
      "eval_logps/rejected": -276.9588623046875,
      "eval_loss": 0.6145145297050476,
      "eval_rewards/accuracies": 0.7023809552192688,
      "eval_rewards/chosen": 0.01473198737949133,
      "eval_rewards/margins": 0.2757962942123413,
      "eval_rewards/rejected": -0.2610643208026886,
      "eval_runtime": 413.5508,
      "eval_samples_per_second": 4.836,
      "eval_steps_per_second": 0.152,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 3.666666666666666e-07,
      "logits/chosen": -2.4004225730895996,
      "logits/rejected": -2.336871385574341,
      "logps/chosen": -265.05352783203125,
      "logps/rejected": -254.93212890625,
      "loss": 0.6078,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.014697420410811901,
      "rewards/margins": 0.27927759289741516,
      "rewards/rejected": -0.29397499561309814,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4e-07,
      "logits/chosen": -2.400813579559326,
      "logits/rejected": -2.317667245864868,
      "logps/chosen": -289.80938720703125,
      "logps/rejected": -268.9970703125,
      "loss": 0.6067,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.11111988872289658,
      "rewards/margins": 0.3709365725517273,
      "rewards/rejected": -0.2598166763782501,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.3333333333333335e-07,
      "logits/chosen": -2.4633655548095703,
      "logits/rejected": -2.370208263397217,
      "logps/chosen": -321.96954345703125,
      "logps/rejected": -289.915771484375,
      "loss": 0.5958,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06524975597858429,
      "rewards/margins": 0.3465055525302887,
      "rewards/rejected": -0.2812557816505432,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -2.4572839736938477,
      "logits/rejected": -2.402984857559204,
      "logps/chosen": -267.6603088378906,
      "logps/rejected": -240.7797088623047,
      "loss": 0.5789,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05029069632291794,
      "rewards/margins": 0.3604304790496826,
      "rewards/rejected": -0.3101397752761841,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 5e-07,
      "logits/chosen": -2.3665835857391357,
      "logits/rejected": -2.3545050621032715,
      "logps/chosen": -304.1943054199219,
      "logps/rejected": -298.508056640625,
      "loss": 0.6084,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.032247237861156464,
      "rewards/margins": 0.392182320356369,
      "rewards/rejected": -0.35993510484695435,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.999323102948654e-07,
      "logits/chosen": -2.453312397003174,
      "logits/rejected": -2.3983371257781982,
      "logps/chosen": -269.2843322753906,
      "logps/rejected": -277.93682861328125,
      "loss": 0.5419,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.10002978146076202,
      "rewards/margins": 0.553473174571991,
      "rewards/rejected": -0.4534434676170349,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.997292778346312e-07,
      "logits/chosen": -2.493710517883301,
      "logits/rejected": -2.4315335750579834,
      "logps/chosen": -300.1785888671875,
      "logps/rejected": -257.55157470703125,
      "loss": 0.562,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0674915537238121,
      "rewards/margins": 0.48717761039733887,
      "rewards/rejected": -0.5546691417694092,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.99391012564956e-07,
      "logits/chosen": -2.3834595680236816,
      "logits/rejected": -2.3311541080474854,
      "logps/chosen": -271.11114501953125,
      "logps/rejected": -289.841552734375,
      "loss": 0.5335,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.27489790320396423,
      "rewards/margins": 0.6237329244613647,
      "rewards/rejected": -0.8986309170722961,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.989176976624511e-07,
      "logits/chosen": -2.546043634414673,
      "logits/rejected": -2.4504528045654297,
      "logps/chosen": -307.6429138183594,
      "logps/rejected": -274.8311462402344,
      "loss": 0.5298,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.14119036495685577,
      "rewards/margins": 0.8371874094009399,
      "rewards/rejected": -0.6959971189498901,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.983095894354857e-07,
      "logits/chosen": -2.489591121673584,
      "logits/rejected": -2.4581377506256104,
      "logps/chosen": -268.5490417480469,
      "logps/rejected": -291.10986328125,
      "loss": 0.5608,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1413675844669342,
      "rewards/margins": 0.6023878455162048,
      "rewards/rejected": -0.7437554597854614,
      "step": 200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -2.4285523891448975,
      "eval_logits/rejected": -2.3805854320526123,
      "eval_logps/chosen": -298.1169128417969,
      "eval_logps/rejected": -287.88702392578125,
      "eval_loss": 0.5506500005722046,
      "eval_rewards/accuracies": 0.7400793433189392,
      "eval_rewards/chosen": -0.08984197676181793,
      "eval_rewards/margins": 0.7176319360733032,
      "eval_rewards/rejected": -0.80747389793396,
      "eval_runtime": 409.3127,
      "eval_samples_per_second": 4.886,
      "eval_steps_per_second": 0.154,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.975670171853925e-07,
      "logits/chosen": -2.48987078666687,
      "logits/rejected": -2.4578893184661865,
      "logps/chosen": -309.9327087402344,
      "logps/rejected": -268.804443359375,
      "loss": 0.5555,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.00010260194540023804,
      "rewards/margins": 0.6477264165878296,
      "rewards/rejected": -0.6476237773895264,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.966903830281448e-07,
      "logits/chosen": -2.5101141929626465,
      "logits/rejected": -2.5068182945251465,
      "logps/chosen": -302.55804443359375,
      "logps/rejected": -306.7073059082031,
      "loss": 0.5371,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.022780220955610275,
      "rewards/margins": 0.6337054371833801,
      "rewards/rejected": -0.6109251379966736,
      "step": 220
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.956801616766034e-07,
      "logits/chosen": -2.477936267852783,
      "logits/rejected": -2.4627578258514404,
      "logps/chosen": -334.6808776855469,
      "logps/rejected": -300.4921569824219,
      "loss": 0.5438,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.660649299621582,
      "rewards/margins": 0.6745242476463318,
      "rewards/rejected": -1.3351736068725586,
      "step": 230
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -2.4449329376220703,
      "logits/rejected": -2.3937153816223145,
      "logps/chosen": -280.98468017578125,
      "logps/rejected": -243.7540283203125,
      "loss": 0.5362,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.5287132263183594,
      "rewards/margins": 0.6548066139221191,
      "rewards/rejected": -1.1835198402404785,
      "step": 240
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.932612176449559e-07,
      "logits/chosen": -2.3831703662872314,
      "logits/rejected": -2.3532094955444336,
      "logps/chosen": -308.6506042480469,
      "logps/rejected": -295.2610778808594,
      "loss": 0.5517,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.20035383105278015,
      "rewards/margins": 0.6892730593681335,
      "rewards/rejected": -0.8896268606185913,
      "step": 250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.918538048657159e-07,
      "logits/chosen": -2.381070852279663,
      "logits/rejected": -2.356685161590576,
      "logps/chosen": -260.6903076171875,
      "logps/rejected": -273.7911682128906,
      "loss": 0.5333,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3518093228340149,
      "rewards/margins": 0.6164388060569763,
      "rewards/rejected": -0.9682480692863464,
      "step": 260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.903154239845797e-07,
      "logits/chosen": -2.385469436645508,
      "logits/rejected": -2.3523507118225098,
      "logps/chosen": -312.2256774902344,
      "logps/rejected": -312.83526611328125,
      "loss": 0.5092,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.35005810856819153,
      "rewards/margins": 0.9718685150146484,
      "rewards/rejected": -1.3219265937805176,
      "step": 270
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.88646908061933e-07,
      "logits/chosen": -2.4627230167388916,
      "logits/rejected": -2.418569564819336,
      "logps/chosen": -317.95703125,
      "logps/rejected": -289.7148742675781,
      "loss": 0.5104,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.29435399174690247,
      "rewards/margins": 0.9088577032089233,
      "rewards/rejected": -1.2032116651535034,
      "step": 280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.868491606285823e-07,
      "logits/chosen": -2.4506328105926514,
      "logits/rejected": -2.3806400299072266,
      "logps/chosen": -302.1065673828125,
      "logps/rejected": -251.01498413085938,
      "loss": 0.5353,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.43174147605895996,
      "rewards/margins": 0.6914370656013489,
      "rewards/rejected": -1.123178482055664,
      "step": 290
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": -2.431826591491699,
      "logits/rejected": -2.354094982147217,
      "logps/chosen": -268.3138122558594,
      "logps/rejected": -259.5442199707031,
      "loss": 0.4934,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.18001610040664673,
      "rewards/margins": 0.8297437429428101,
      "rewards/rejected": -1.009759783744812,
      "step": 300
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -2.3849596977233887,
      "eval_logits/rejected": -2.3416402339935303,
      "eval_logps/chosen": -299.6116638183594,
      "eval_logps/rejected": -292.5220947265625,
      "eval_loss": 0.5225438475608826,
      "eval_rewards/accuracies": 0.7579365372657776,
      "eval_rewards/chosen": -0.16457918286323547,
      "eval_rewards/margins": 0.8746477365493774,
      "eval_rewards/rejected": -1.03922700881958,
      "eval_runtime": 407.614,
      "eval_samples_per_second": 4.907,
      "eval_steps_per_second": 0.155,
      "step": 300
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.828699347315356e-07,
      "logits/chosen": -2.3300023078918457,
      "logits/rejected": -2.291306972503662,
      "logps/chosen": -281.8946228027344,
      "logps/rejected": -328.680908203125,
      "loss": 0.4757,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.23267221450805664,
      "rewards/margins": 1.1307260990142822,
      "rewards/rejected": -1.3633983135223389,
      "step": 310
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.806906110888606e-07,
      "logits/chosen": -2.2734484672546387,
      "logits/rejected": -2.2220654487609863,
      "logps/chosen": -344.95184326171875,
      "logps/rejected": -317.2240295410156,
      "loss": 0.5251,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.40918412804603577,
      "rewards/margins": 1.0021181106567383,
      "rewards/rejected": -1.4113022089004517,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.783863644106502e-07,
      "logits/chosen": -2.2840628623962402,
      "logits/rejected": -2.2298922538757324,
      "logps/chosen": -281.0782775878906,
      "logps/rejected": -242.35263061523438,
      "loss": 0.553,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.017911773175001144,
      "rewards/margins": 0.6968544721603394,
      "rewards/rejected": -0.714766263961792,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.759584424871301e-07,
      "logits/chosen": -2.270648717880249,
      "logits/rejected": -2.2538352012634277,
      "logps/chosen": -297.785888671875,
      "logps/rejected": -256.94305419921875,
      "loss": 0.5741,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.040852271020412445,
      "rewards/margins": 0.6195092797279358,
      "rewards/rejected": -0.5786570310592651,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.7340816008085305e-07,
      "logits/chosen": -2.3321385383605957,
      "logits/rejected": -2.301793336868286,
      "logps/chosen": -321.775390625,
      "logps/rejected": -314.6920471191406,
      "loss": 0.5205,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.3098350465297699,
      "rewards/margins": 0.9677619934082031,
      "rewards/rejected": -1.2775970697402954,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.707368982147317e-07,
      "logits/chosen": -2.460111379623413,
      "logits/rejected": -2.429689884185791,
      "logps/chosen": -289.2377014160156,
      "logps/rejected": -296.45123291015625,
      "loss": 0.5145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8857641220092773,
      "rewards/margins": 0.7435467839241028,
      "rewards/rejected": -1.6293108463287354,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.6794610342419056e-07,
      "logits/chosen": -2.403916120529175,
      "logits/rejected": -2.338923931121826,
      "logps/chosen": -287.01434326171875,
      "logps/rejected": -307.77728271484375,
      "loss": 0.4818,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.28949111700057983,
      "rewards/margins": 1.0141589641571045,
      "rewards/rejected": -1.303650140762329,
      "step": 370
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.650372869738414e-07,
      "logits/chosen": -2.4299778938293457,
      "logits/rejected": -2.404153347015381,
      "logps/chosen": -310.12152099609375,
      "logps/rejected": -294.5881652832031,
      "loss": 0.5046,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.4136735796928406,
      "rewards/margins": 0.9840124249458313,
      "rewards/rejected": -1.3976860046386719,
      "step": 380
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.6201202403910643e-07,
      "logits/chosen": -2.3376293182373047,
      "logits/rejected": -2.289973020553589,
      "logps/chosen": -299.35546875,
      "logps/rejected": -284.88323974609375,
      "loss": 0.5184,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.6497005224227905,
      "rewards/margins": 1.0391151905059814,
      "rewards/rejected": -1.688815712928772,
      "step": 390
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.588719528532341e-07,
      "logits/chosen": -2.458130359649658,
      "logits/rejected": -2.4228129386901855,
      "logps/chosen": -315.5902404785156,
      "logps/rejected": -292.9128112792969,
      "loss": 0.4812,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5732877254486084,
      "rewards/margins": 0.7860868573188782,
      "rewards/rejected": -1.359374761581421,
      "step": 400
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -2.369720458984375,
      "eval_logits/rejected": -2.3284802436828613,
      "eval_logps/chosen": -300.5794982910156,
      "eval_logps/rejected": -295.33331298828125,
      "eval_loss": 0.5147603154182434,
      "eval_rewards/accuracies": 0.7599206566810608,
      "eval_rewards/chosen": -0.21297024190425873,
      "eval_rewards/margins": 0.9668172597885132,
      "eval_rewards/rejected": -1.179787516593933,
      "eval_runtime": 407.8456,
      "eval_samples_per_second": 4.904,
      "eval_steps_per_second": 0.154,
      "step": 400
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.5561877382016553e-07,
      "logits/chosen": -2.4733567237854004,
      "logits/rejected": -2.429727792739868,
      "logps/chosen": -300.5076599121094,
      "logps/rejected": -275.7524108886719,
      "loss": 0.5144,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.15985186398029327,
      "rewards/margins": 0.7756703495979309,
      "rewards/rejected": -0.935522198677063,
      "step": 410
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.5225424859373684e-07,
      "logits/chosen": -2.3798115253448486,
      "logits/rejected": -2.3424534797668457,
      "logps/chosen": -243.74472045898438,
      "logps/rejected": -266.5513916015625,
      "loss": 0.5302,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.005200001411139965,
      "rewards/margins": 0.7427350878715515,
      "rewards/rejected": -0.7479350566864014,
      "step": 420
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.487801991237119e-07,
      "logits/chosen": -2.441993236541748,
      "logits/rejected": -2.414069890975952,
      "logps/chosen": -303.46893310546875,
      "logps/rejected": -286.26470947265625,
      "loss": 0.5387,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12445823103189468,
      "rewards/margins": 0.7546092867851257,
      "rewards/rejected": -0.8790675401687622,
      "step": 430
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.451985066691648e-07,
      "logits/chosen": -2.444979429244995,
      "logits/rejected": -2.402029514312744,
      "logps/chosen": -300.88037109375,
      "logps/rejected": -275.1875915527344,
      "loss": 0.4965,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5577610731124878,
      "rewards/margins": 0.7658098340034485,
      "rewards/rejected": -1.323570966720581,
      "step": 440
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": -2.290574550628662,
      "logits/rejected": -2.2670164108276367,
      "logps/chosen": -236.2515106201172,
      "logps/rejected": -255.0359344482422,
      "loss": 0.5183,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5639761090278625,
      "rewards/margins": 0.8503490686416626,
      "rewards/rejected": -1.4143251180648804,
      "step": 450
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.377200082453748e-07,
      "logits/chosen": -2.467921495437622,
      "logits/rejected": -2.422136068344116,
      "logps/chosen": -329.4405822753906,
      "logps/rejected": -305.3108825683594,
      "loss": 0.4962,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.41245952248573303,
      "rewards/margins": 0.9212630987167358,
      "rewards/rejected": -1.333722710609436,
      "step": 460
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.3382725201495717e-07,
      "logits/chosen": -2.3804707527160645,
      "logits/rejected": -2.3205363750457764,
      "logps/chosen": -282.99945068359375,
      "logps/rejected": -292.9880065917969,
      "loss": 0.4927,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5624107718467712,
      "rewards/margins": 0.8573344349861145,
      "rewards/rejected": -1.4197450876235962,
      "step": 470
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.2983495008466273e-07,
      "logits/chosen": -2.302927255630493,
      "logits/rejected": -2.272263765335083,
      "logps/chosen": -271.0894470214844,
      "logps/rejected": -286.90118408203125,
      "loss": 0.5345,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.5512509346008301,
      "rewards/margins": 0.8141476511955261,
      "rewards/rejected": -1.365398645401001,
      "step": 480
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.2574526435641546e-07,
      "logits/chosen": -2.311328172683716,
      "logits/rejected": -2.276010274887085,
      "logps/chosen": -285.27960205078125,
      "logps/rejected": -270.05401611328125,
      "loss": 0.5116,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.49702826142311096,
      "rewards/margins": 0.8841336369514465,
      "rewards/rejected": -1.3811619281768799,
      "step": 490
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.2156040946718343e-07,
      "logits/chosen": -2.3711423873901367,
      "logits/rejected": -2.286292552947998,
      "logps/chosen": -304.9607238769531,
      "logps/rejected": -297.2706604003906,
      "loss": 0.5217,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5594313740730286,
      "rewards/margins": 1.1461693048477173,
      "rewards/rejected": -1.7056007385253906,
      "step": 500
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.3431873321533203,
      "eval_logits/rejected": -2.3073697090148926,
      "eval_logps/chosen": -299.8136291503906,
      "eval_logps/rejected": -294.8787841796875,
      "eval_loss": 0.5093801617622375,
      "eval_rewards/accuracies": 0.7599206566810608,
      "eval_rewards/chosen": -0.17467793822288513,
      "eval_rewards/margins": 0.982383131980896,
      "eval_rewards/rejected": -1.157060980796814,
      "eval_runtime": 409.4153,
      "eval_samples_per_second": 4.885,
      "eval_steps_per_second": 0.154,
      "step": 500
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.172826515897145e-07,
      "logits/chosen": -2.380160331726074,
      "logits/rejected": -2.3342180252075195,
      "logps/chosen": -285.99005126953125,
      "logps/rejected": -262.095703125,
      "loss": 0.4819,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.1991649866104126,
      "rewards/margins": 0.9838080406188965,
      "rewards/rejected": -1.1829731464385986,
      "step": 510
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.129143072053638e-07,
      "logits/chosen": -2.430504322052002,
      "logits/rejected": -2.3967652320861816,
      "logps/chosen": -295.1175842285156,
      "logps/rejected": -298.79034423828125,
      "loss": 0.4879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2341586798429489,
      "rewards/margins": 0.9701293110847473,
      "rewards/rejected": -1.2042880058288574,
      "step": 520
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.084577418496775e-07,
      "logits/chosen": -2.459068775177002,
      "logits/rejected": -2.4153661727905273,
      "logps/chosen": -303.06683349609375,
      "logps/rejected": -333.80609130859375,
      "loss": 0.4868,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4716320037841797,
      "rewards/margins": 1.0053110122680664,
      "rewards/rejected": -1.4769428968429565,
      "step": 530
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.039153688314145e-07,
      "logits/chosen": -2.4637537002563477,
      "logits/rejected": -2.4269044399261475,
      "logps/chosen": -304.27166748046875,
      "logps/rejected": -302.20477294921875,
      "loss": 0.4986,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.45887041091918945,
      "rewards/margins": 1.0204750299453735,
      "rewards/rejected": -1.4793453216552734,
      "step": 540
    },
    {
      "epoch": 0.58,
      "learning_rate": 3.9928964792569654e-07,
      "logits/chosen": -2.4167208671569824,
      "logits/rejected": -2.3573527336120605,
      "logps/chosen": -303.3789978027344,
      "logps/rejected": -296.81060791015625,
      "loss": 0.5343,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.59644615650177,
      "rewards/margins": 0.9994708895683289,
      "rewards/rejected": -1.595916986465454,
      "step": 550
    },
    {
      "epoch": 0.59,
      "learning_rate": 3.945830840419966e-07,
      "logits/chosen": -2.3017094135284424,
      "logits/rejected": -2.2486953735351562,
      "logps/chosen": -298.501708984375,
      "logps/rejected": -250.04556274414062,
      "loss": 0.5037,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.4263156056404114,
      "rewards/margins": 0.9166458249092102,
      "rewards/rejected": -1.3429615497589111,
      "step": 560
    },
    {
      "epoch": 0.6,
      "learning_rate": 3.8979822586768666e-07,
      "logits/chosen": -2.308485507965088,
      "logits/rejected": -2.259333610534668,
      "logps/chosen": -274.44329833984375,
      "logps/rejected": -297.6800537109375,
      "loss": 0.5011,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.19708169996738434,
      "rewards/margins": 0.9468144178390503,
      "rewards/rejected": -1.1438961029052734,
      "step": 570
    },
    {
      "epoch": 0.61,
      "learning_rate": 3.849376644878782e-07,
      "logits/chosen": -2.3406989574432373,
      "logits/rejected": -2.2872297763824463,
      "logps/chosen": -293.8192443847656,
      "logps/rejected": -292.8212585449219,
      "loss": 0.5012,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.24491167068481445,
      "rewards/margins": 1.1491771936416626,
      "rewards/rejected": -1.394088864326477,
      "step": 580
    },
    {
      "epoch": 0.62,
      "learning_rate": 3.800040319823038e-07,
      "logits/chosen": -2.2375845909118652,
      "logits/rejected": -2.21081280708313,
      "logps/chosen": -281.64678955078125,
      "logps/rejected": -281.1907958984375,
      "loss": 0.5189,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.33811599016189575,
      "rewards/margins": 0.640630841255188,
      "rewards/rejected": -0.9787468910217285,
      "step": 590
    },
    {
      "epoch": 0.63,
      "learning_rate": 3.75e-07,
      "logits/chosen": -2.336167812347412,
      "logits/rejected": -2.3122599124908447,
      "logps/chosen": -316.1159362792969,
      "logps/rejected": -348.52728271484375,
      "loss": 0.5069,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.08971457183361053,
      "rewards/margins": 1.1014090776443481,
      "rewards/rejected": -1.0116945505142212,
      "step": 600
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.2853615283966064,
      "eval_logits/rejected": -2.2443532943725586,
      "eval_logps/chosen": -297.127197265625,
      "eval_logps/rejected": -292.7251281738281,
      "eval_loss": 0.5037240982055664,
      "eval_rewards/accuracies": 0.7658730149269104,
      "eval_rewards/chosen": -0.04035760462284088,
      "eval_rewards/margins": 1.009021520614624,
      "eval_rewards/rejected": -1.0493791103363037,
      "eval_runtime": 411.3039,
      "eval_samples_per_second": 4.863,
      "eval_steps_per_second": 0.153,
      "step": 600
    },
    {
      "epoch": 0.64,
      "learning_rate": 3.699282783125616e-07,
      "logits/chosen": -2.29797625541687,
      "logits/rejected": -2.2567286491394043,
      "logps/chosen": -333.64312744140625,
      "logps/rejected": -276.2323913574219,
      "loss": 0.4939,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5196839570999146,
      "rewards/margins": 0.9884433746337891,
      "rewards/rejected": -1.508127212524414,
      "step": 610
    },
    {
      "epoch": 0.65,
      "learning_rate": 3.647916133467529e-07,
      "logits/chosen": -2.3571763038635254,
      "logits/rejected": -2.2932493686676025,
      "logps/chosen": -308.9863586425781,
      "logps/rejected": -301.33367919921875,
      "loss": 0.4919,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.0287619829177856,
      "rewards/margins": 0.9334309697151184,
      "rewards/rejected": -1.9621927738189697,
      "step": 620
    },
    {
      "epoch": 0.66,
      "learning_rate": 3.595927866972693e-07,
      "logits/chosen": -2.352038860321045,
      "logits/rejected": -2.2862563133239746,
      "logps/chosen": -293.9407958984375,
      "logps/rejected": -288.50885009765625,
      "loss": 0.4733,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5753384828567505,
      "rewards/margins": 1.169237494468689,
      "rewards/rejected": -1.744576096534729,
      "step": 630
    },
    {
      "epoch": 0.67,
      "learning_rate": 3.5433461362045447e-07,
      "logits/chosen": -2.3236405849456787,
      "logits/rejected": -2.302849054336548,
      "logps/chosen": -262.9497375488281,
      "logps/rejected": -284.64642333984375,
      "loss": 0.4962,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.15619780123233795,
      "rewards/margins": 1.1346657276153564,
      "rewards/rejected": -1.2908635139465332,
      "step": 640
    },
    {
      "epoch": 0.68,
      "learning_rate": 3.490199415097892e-07,
      "logits/chosen": -2.3758909702301025,
      "logits/rejected": -2.300546169281006,
      "logps/chosen": -324.71881103515625,
      "logps/rejected": -303.84881591796875,
      "loss": 0.494,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 0.004534644540399313,
      "rewards/margins": 1.049611210823059,
      "rewards/rejected": -1.045076608657837,
      "step": 650
    },
    {
      "epoch": 0.69,
      "learning_rate": 3.43651648353978e-07,
      "logits/chosen": -2.4268946647644043,
      "logits/rejected": -2.3779337406158447,
      "logps/chosen": -320.46429443359375,
      "logps/rejected": -281.7370910644531,
      "loss": 0.5142,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.28596049547195435,
      "rewards/margins": 0.7663468718528748,
      "rewards/rejected": -1.052307367324829,
      "step": 660
    },
    {
      "epoch": 0.7,
      "learning_rate": 3.3823264117846717e-07,
      "logits/chosen": -2.353480100631714,
      "logits/rejected": -2.3156867027282715,
      "logps/chosen": -294.9992980957031,
      "logps/rejected": -279.8269958496094,
      "loss": 0.5042,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3519328236579895,
      "rewards/margins": 1.0733650922775269,
      "rewards/rejected": -1.4252979755401611,
      "step": 670
    },
    {
      "epoch": 0.71,
      "learning_rate": 3.327658544712395e-07,
      "logits/chosen": -2.349534273147583,
      "logits/rejected": -2.2964000701904297,
      "logps/chosen": -302.86798095703125,
      "logps/rejected": -278.29827880859375,
      "loss": 0.5219,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.47534093260765076,
      "rewards/margins": 0.663826584815979,
      "rewards/rejected": -1.1391674280166626,
      "step": 680
    },
    {
      "epoch": 0.72,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.3173270225524902,
      "logits/rejected": -2.2520382404327393,
      "logps/chosen": -266.728515625,
      "logps/rejected": -234.1517333984375,
      "loss": 0.5044,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2846493721008301,
      "rewards/margins": 0.9897111058235168,
      "rewards/rejected": -1.2743605375289917,
      "step": 690
    },
    {
      "epoch": 0.73,
      "learning_rate": 3.2170080817777257e-07,
      "logits/chosen": -2.3343617916107178,
      "logits/rejected": -2.2195441722869873,
      "logps/chosen": -297.25250244140625,
      "logps/rejected": -286.7340393066406,
      "loss": 0.4582,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5840951800346375,
      "rewards/margins": 1.0429940223693848,
      "rewards/rejected": -1.6270891427993774,
      "step": 700
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -2.289731979370117,
      "eval_logits/rejected": -2.246857166290283,
      "eval_logps/chosen": -308.9957580566406,
      "eval_logps/rejected": -306.2008056640625,
      "eval_loss": 0.500346839427948,
      "eval_rewards/accuracies": 0.7599206566810608,
      "eval_rewards/chosen": -0.6337829232215881,
      "eval_rewards/margins": 1.0893793106079102,
      "eval_rewards/rejected": -1.7231621742248535,
      "eval_runtime": 438.5736,
      "eval_samples_per_second": 4.56,
      "eval_steps_per_second": 0.144,
      "step": 700
    },
    {
      "epoch": 0.74,
      "learning_rate": 3.1610854050930057e-07,
      "logits/chosen": -2.3422415256500244,
      "logits/rejected": -2.2789344787597656,
      "logps/chosen": -298.95867919921875,
      "logps/rejected": -271.91290283203125,
      "loss": 0.474,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5943951606750488,
      "rewards/margins": 0.9698866009712219,
      "rewards/rejected": -1.5642818212509155,
      "step": 710
    },
    {
      "epoch": 0.75,
      "learning_rate": 3.104804738999169e-07,
      "logits/chosen": -2.296921968460083,
      "logits/rejected": -2.277675151824951,
      "logps/chosen": -288.68511962890625,
      "logps/rejected": -294.5284729003906,
      "loss": 0.4741,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.4995554983615875,
      "rewards/margins": 1.2392946481704712,
      "rewards/rejected": -1.7388503551483154,
      "step": 720
    },
    {
      "epoch": 0.76,
      "learning_rate": 3.048196560469758e-07,
      "logits/chosen": -2.320403575897217,
      "logits/rejected": -2.276542901992798,
      "logps/chosen": -317.94012451171875,
      "logps/rejected": -283.49920654296875,
      "loss": 0.4819,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.4403966963291168,
      "rewards/margins": 1.0142490863800049,
      "rewards/rejected": -1.4546458721160889,
      "step": 730
    },
    {
      "epoch": 0.77,
      "learning_rate": 2.991291523832075e-07,
      "logits/chosen": -2.3161144256591797,
      "logits/rejected": -2.2607274055480957,
      "logps/chosen": -326.93829345703125,
      "logps/rejected": -328.2031555175781,
      "loss": 0.4746,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6947178840637207,
      "rewards/margins": 1.2727077007293701,
      "rewards/rejected": -1.9674255847930908,
      "step": 740
    },
    {
      "epoch": 0.79,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -2.292912006378174,
      "logits/rejected": -2.2443642616271973,
      "logps/chosen": -282.31536865234375,
      "logps/rejected": -312.34735107421875,
      "loss": 0.4777,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.9387601017951965,
      "rewards/margins": 1.1481364965438843,
      "rewards/rejected": -2.0868964195251465,
      "step": 750
    },
    {
      "epoch": 0.8,
      "learning_rate": 2.8767142806237077e-07,
      "logits/chosen": -2.369844913482666,
      "logits/rejected": -2.368522882461548,
      "logps/chosen": -308.49700927734375,
      "logps/rejected": -300.10455322265625,
      "loss": 0.4665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.539741039276123,
      "rewards/margins": 1.0256460905075073,
      "rewards/rejected": -1.5653870105743408,
      "step": 760
    },
    {
      "epoch": 0.81,
      "learning_rate": 2.819104119651487e-07,
      "logits/chosen": -2.3911471366882324,
      "logits/rejected": -2.3795692920684814,
      "logps/chosen": -315.2966613769531,
      "logps/rejected": -321.0699157714844,
      "loss": 0.5096,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.39289337396621704,
      "rewards/margins": 1.0190023183822632,
      "rewards/rejected": -1.411895751953125,
      "step": 770
    },
    {
      "epoch": 0.82,
      "learning_rate": 2.761321158169134e-07,
      "logits/chosen": -2.3824098110198975,
      "logits/rejected": -2.333876132965088,
      "logps/chosen": -309.57403564453125,
      "logps/rejected": -271.635009765625,
      "loss": 0.4965,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.31998205184936523,
      "rewards/margins": 1.0204508304595947,
      "rewards/rejected": -1.3404327630996704,
      "step": 780
    },
    {
      "epoch": 0.83,
      "learning_rate": 2.703396686669646e-07,
      "logits/chosen": -2.3626012802124023,
      "logits/rejected": -2.336550235748291,
      "logps/chosen": -282.4324035644531,
      "logps/rejected": -289.73907470703125,
      "loss": 0.4969,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.40831103920936584,
      "rewards/margins": 0.9678645133972168,
      "rewards/rejected": -1.3761756420135498,
      "step": 790
    },
    {
      "epoch": 0.84,
      "learning_rate": 2.6453620722761895e-07,
      "logits/chosen": -2.2754549980163574,
      "logits/rejected": -2.227632999420166,
      "logps/chosen": -324.36065673828125,
      "logps/rejected": -315.53265380859375,
      "loss": 0.457,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.40555134415626526,
      "rewards/margins": 1.2698910236358643,
      "rewards/rejected": -1.6754423379898071,
      "step": 800
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -2.3341588973999023,
      "eval_logits/rejected": -2.292834520339966,
      "eval_logps/chosen": -306.1227722167969,
      "eval_logps/rejected": -303.84637451171875,
      "eval_loss": 0.4906502664089203,
      "eval_rewards/accuracies": 0.7638888955116272,
      "eval_rewards/chosen": -0.4901339113712311,
      "eval_rewards/margins": 1.1153074502944946,
      "eval_rewards/rejected": -1.6054412126541138,
      "eval_runtime": 442.4535,
      "eval_samples_per_second": 4.52,
      "eval_steps_per_second": 0.142,
      "step": 800
    },
    {
      "epoch": 0.85,
      "learning_rate": 2.5872487417562527e-07,
      "logits/chosen": -2.3688154220581055,
      "logits/rejected": -2.2756457328796387,
      "logps/chosen": -302.0091552734375,
      "logps/rejected": -267.61712646484375,
      "loss": 0.5048,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.739054799079895,
      "rewards/margins": 0.9176353216171265,
      "rewards/rejected": -1.656690001487732,
      "step": 810
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.5290881645034926e-07,
      "logits/chosen": -2.371279239654541,
      "logits/rejected": -2.3216042518615723,
      "logps/chosen": -307.4602966308594,
      "logps/rejected": -276.8441162109375,
      "loss": 0.5156,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4735354781150818,
      "rewards/margins": 1.1825616359710693,
      "rewards/rejected": -1.656097173690796,
      "step": 820
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4709118354965077e-07,
      "logits/chosen": -2.3698923587799072,
      "logits/rejected": -2.3812479972839355,
      "logps/chosen": -283.11395263671875,
      "logps/rejected": -314.9549865722656,
      "loss": 0.4728,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.5311114192008972,
      "rewards/margins": 0.8778692483901978,
      "rewards/rejected": -1.4089806079864502,
      "step": 830
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.412751258243748e-07,
      "logits/chosen": -2.377016305923462,
      "logits/rejected": -2.3782734870910645,
      "logps/chosen": -294.67608642578125,
      "logps/rejected": -310.6152038574219,
      "loss": 0.4955,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6087425351142883,
      "rewards/margins": 0.8135608434677124,
      "rewards/rejected": -1.4223034381866455,
      "step": 840
    },
    {
      "epoch": 0.89,
      "learning_rate": 2.3546379277238103e-07,
      "logits/chosen": -2.422614574432373,
      "logits/rejected": -2.39127779006958,
      "logps/chosen": -313.9159240722656,
      "logps/rejected": -270.3652648925781,
      "loss": 0.4874,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.7056568264961243,
      "rewards/margins": 0.9642340540885925,
      "rewards/rejected": -1.6698909997940063,
      "step": 850
    },
    {
      "epoch": 0.9,
      "learning_rate": 2.2966033133303545e-07,
      "logits/chosen": -2.378150463104248,
      "logits/rejected": -2.3483357429504395,
      "logps/chosen": -326.0942687988281,
      "logps/rejected": -330.057373046875,
      "loss": 0.4994,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.8599470257759094,
      "rewards/margins": 0.9078429341316223,
      "rewards/rejected": -1.7677898406982422,
      "step": 860
    },
    {
      "epoch": 0.91,
      "learning_rate": 2.2386788418308665e-07,
      "logits/chosen": -2.4225101470947266,
      "logits/rejected": -2.4114060401916504,
      "logps/chosen": -266.55108642578125,
      "logps/rejected": -283.1296081542969,
      "loss": 0.501,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5755284428596497,
      "rewards/margins": 1.0859739780426025,
      "rewards/rejected": -1.6615022420883179,
      "step": 870
    },
    {
      "epoch": 0.92,
      "learning_rate": 2.1808958803485133e-07,
      "logits/chosen": -2.483870506286621,
      "logits/rejected": -2.426514148712158,
      "logps/chosen": -305.41119384765625,
      "logps/rejected": -296.4634704589844,
      "loss": 0.4761,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.4832102358341217,
      "rewards/margins": 0.9984723925590515,
      "rewards/rejected": -1.481682538986206,
      "step": 880
    },
    {
      "epoch": 0.93,
      "learning_rate": 2.123285719376292e-07,
      "logits/chosen": -2.4902679920196533,
      "logits/rejected": -2.4276280403137207,
      "logps/chosen": -306.55987548828125,
      "logps/rejected": -308.28216552734375,
      "loss": 0.4792,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.5795844793319702,
      "rewards/margins": 1.21144700050354,
      "rewards/rejected": -1.7910315990447998,
      "step": 890
    },
    {
      "epoch": 0.94,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -2.3825860023498535,
      "logits/rejected": -2.32081937789917,
      "logps/chosen": -300.0443115234375,
      "logps/rejected": -298.1470642089844,
      "loss": 0.4723,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6022388339042664,
      "rewards/margins": 1.1292412281036377,
      "rewards/rejected": -1.7314800024032593,
      "step": 900
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -2.3762104511260986,
      "eval_logits/rejected": -2.3354876041412354,
      "eval_logps/chosen": -305.15618896484375,
      "eval_logps/rejected": -302.8718566894531,
      "eval_loss": 0.4933333098888397,
      "eval_rewards/accuracies": 0.7658730149269104,
      "eval_rewards/chosen": -0.4418059289455414,
      "eval_rewards/margins": 1.114911437034607,
      "eval_rewards/rejected": -1.5567171573638916,
      "eval_runtime": 427.8237,
      "eval_samples_per_second": 4.675,
      "eval_steps_per_second": 0.147,
      "step": 900
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.0087084761679243e-07,
      "logits/chosen": -2.433973789215088,
      "logits/rejected": -2.3652501106262207,
      "logps/chosen": -312.04620361328125,
      "logps/rejected": -285.4141540527344,
      "loss": 0.4747,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3190104365348816,
      "rewards/margins": 1.0960520505905151,
      "rewards/rejected": -1.4150625467300415,
      "step": 910
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9518034395302412e-07,
      "logits/chosen": -2.3947741985321045,
      "logits/rejected": -2.349290370941162,
      "logps/chosen": -300.0207214355469,
      "logps/rejected": -293.3101501464844,
      "loss": 0.4644,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2700619399547577,
      "rewards/margins": 1.2211670875549316,
      "rewards/rejected": -1.4912290573120117,
      "step": 920
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.895195261000831e-07,
      "logits/chosen": -2.426356792449951,
      "logits/rejected": -2.363300323486328,
      "logps/chosen": -273.56988525390625,
      "logps/rejected": -281.79144287109375,
      "loss": 0.4792,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.19313231110572815,
      "rewards/margins": 1.6159725189208984,
      "rewards/rejected": -1.8091049194335938,
      "step": 930
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.8389145949069951e-07,
      "logits/chosen": -2.504002094268799,
      "logits/rejected": -2.434199810028076,
      "logps/chosen": -312.13922119140625,
      "logps/rejected": -303.17279052734375,
      "loss": 0.4749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8358801603317261,
      "rewards/margins": 1.0024197101593018,
      "rewards/rejected": -1.8382999897003174,
      "step": 940
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.782991918222275e-07,
      "logits/chosen": -2.4335973262786865,
      "logits/rejected": -2.371967315673828,
      "logps/chosen": -297.8493957519531,
      "logps/rejected": -308.2081604003906,
      "loss": 0.5115,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.042378544807434,
      "rewards/margins": 0.9967982172966003,
      "rewards/rejected": -2.0391769409179688,
      "step": 950
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.7274575140626315e-07,
      "logits/chosen": -2.4119739532470703,
      "logits/rejected": -2.3814332485198975,
      "logps/chosen": -291.4102478027344,
      "logps/rejected": -332.5351257324219,
      "loss": 0.3924,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.6064465045928955,
      "rewards/margins": 1.3707728385925293,
      "rewards/rejected": -1.9772193431854248,
      "step": 960
    },
    {
      "epoch": 1.02,
      "learning_rate": 1.672341455287605e-07,
      "logits/chosen": -2.4356229305267334,
      "logits/rejected": -2.380049228668213,
      "logps/chosen": -317.14300537109375,
      "logps/rejected": -315.25616455078125,
      "loss": 0.2962,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3146964907646179,
      "rewards/margins": 1.8185898065567017,
      "rewards/rejected": -2.133286237716675,
      "step": 970
    },
    {
      "epoch": 1.03,
      "learning_rate": 1.617673588215328e-07,
      "logits/chosen": -2.4142088890075684,
      "logits/rejected": -2.3693430423736572,
      "logps/chosen": -248.46224975585938,
      "logps/rejected": -285.1511535644531,
      "loss": 0.2886,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.18761667609214783,
      "rewards/margins": 1.7726829051971436,
      "rewards/rejected": -1.9602997303009033,
      "step": 980
    },
    {
      "epoch": 1.04,
      "learning_rate": 1.5634835164602196e-07,
      "logits/chosen": -2.4898295402526855,
      "logits/rejected": -2.4521660804748535,
      "logps/chosen": -336.2506103515625,
      "logps/rejected": -302.1678161621094,
      "loss": 0.2961,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21727129817008972,
      "rewards/margins": 1.7004649639129639,
      "rewards/rejected": -1.917736291885376,
      "step": 990
    },
    {
      "epoch": 1.05,
      "learning_rate": 1.5098005849021078e-07,
      "logits/chosen": -2.4500224590301514,
      "logits/rejected": -2.4026894569396973,
      "logps/chosen": -296.7429504394531,
      "logps/rejected": -283.27789306640625,
      "loss": 0.3094,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.8134616017341614,
      "rewards/margins": 1.605277419090271,
      "rewards/rejected": -2.418738842010498,
      "step": 1000
    },
    {
      "epoch": 1.05,
      "eval_logits/chosen": -2.40937876701355,
      "eval_logits/rejected": -2.369751214981079,
      "eval_logps/chosen": -312.38043212890625,
      "eval_logps/rejected": -312.68560791015625,
      "eval_loss": 0.49218976497650146,
      "eval_rewards/accuracies": 0.7638888955116272,
      "eval_rewards/chosen": -0.803017795085907,
      "eval_rewards/margins": 1.2443830966949463,
      "eval_rewards/rejected": -2.04740047454834,
      "eval_runtime": 407.9979,
      "eval_samples_per_second": 4.902,
      "eval_steps_per_second": 0.154,
      "step": 1000
    },
    {
      "epoch": 1.06,
      "learning_rate": 1.4566538637954554e-07,
      "logits/chosen": -2.456105947494507,
      "logits/rejected": -2.391247272491455,
      "logps/chosen": -317.70330810546875,
      "logps/rejected": -292.7303771972656,
      "loss": 0.2867,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.4770506024360657,
      "rewards/margins": 1.7201404571533203,
      "rewards/rejected": -2.1971912384033203,
      "step": 1010
    },
    {
      "epoch": 1.07,
      "learning_rate": 1.404072133027306e-07,
      "logits/chosen": -2.468541383743286,
      "logits/rejected": -2.4047131538391113,
      "logps/chosen": -324.1610107421875,
      "logps/rejected": -310.6138610839844,
      "loss": 0.3011,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3531934320926666,
      "rewards/margins": 1.7022035121917725,
      "rewards/rejected": -2.055396795272827,
      "step": 1020
    },
    {
      "epoch": 1.08,
      "learning_rate": 1.3520838665324703e-07,
      "logits/chosen": -2.451047658920288,
      "logits/rejected": -2.426084280014038,
      "logps/chosen": -289.8973083496094,
      "logps/rejected": -315.23968505859375,
      "loss": 0.2848,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.3617556095123291,
      "rewards/margins": 1.7169090509414673,
      "rewards/rejected": -2.078664779663086,
      "step": 1030
    },
    {
      "epoch": 1.09,
      "learning_rate": 1.3007172168743852e-07,
      "logits/chosen": -2.4811718463897705,
      "logits/rejected": -2.442286968231201,
      "logps/chosen": -315.90509033203125,
      "logps/rejected": -345.96820068359375,
      "loss": 0.2943,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2960956394672394,
      "rewards/margins": 1.6934324502944946,
      "rewards/rejected": -1.9895280599594116,
      "step": 1040
    },
    {
      "epoch": 1.1,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -2.4405837059020996,
      "logits/rejected": -2.38596773147583,
      "logps/chosen": -287.0311279296875,
      "logps/rejected": -303.80535888671875,
      "loss": 0.2866,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.2767558693885803,
      "rewards/margins": 1.9063446521759033,
      "rewards/rejected": -2.183100461959839,
      "step": 1050
    },
    {
      "epoch": 1.11,
      "learning_rate": 1.1999596801769616e-07,
      "logits/chosen": -2.440513849258423,
      "logits/rejected": -2.3295609951019287,
      "logps/chosen": -326.0518493652344,
      "logps/rejected": -333.1617431640625,
      "loss": 0.2744,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.25197187066078186,
      "rewards/margins": 2.045137882232666,
      "rewards/rejected": -2.297109603881836,
      "step": 1060
    },
    {
      "epoch": 1.12,
      "learning_rate": 1.1506233551212185e-07,
      "logits/chosen": -2.414334535598755,
      "logits/rejected": -2.3740944862365723,
      "logps/chosen": -306.10992431640625,
      "logps/rejected": -327.01959228515625,
      "loss": 0.2711,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.261447936296463,
      "rewards/margins": 2.056182384490967,
      "rewards/rejected": -2.3176300525665283,
      "step": 1070
    },
    {
      "epoch": 1.13,
      "learning_rate": 1.1020177413231332e-07,
      "logits/chosen": -2.424375057220459,
      "logits/rejected": -2.3422465324401855,
      "logps/chosen": -267.51129150390625,
      "logps/rejected": -272.04876708984375,
      "loss": 0.2992,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.5566546320915222,
      "rewards/margins": 1.5810120105743408,
      "rewards/rejected": -2.1376664638519287,
      "step": 1080
    },
    {
      "epoch": 1.14,
      "learning_rate": 1.0541691595800336e-07,
      "logits/chosen": -2.4174036979675293,
      "logits/rejected": -2.417484760284424,
      "logps/chosen": -288.2060852050781,
      "logps/rejected": -316.9212341308594,
      "loss": 0.2776,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.14382892847061157,
      "rewards/margins": 1.983428716659546,
      "rewards/rejected": -2.127257823944092,
      "step": 1090
    },
    {
      "epoch": 1.15,
      "learning_rate": 1.007103520743035e-07,
      "logits/chosen": -2.4386072158813477,
      "logits/rejected": -2.383044719696045,
      "logps/chosen": -316.8988342285156,
      "logps/rejected": -311.27642822265625,
      "loss": 0.2725,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.27822011709213257,
      "rewards/margins": 1.802807092666626,
      "rewards/rejected": -2.081026792526245,
      "step": 1100
    },
    {
      "epoch": 1.15,
      "eval_logits/chosen": -2.3785088062286377,
      "eval_logits/rejected": -2.3381552696228027,
      "eval_logps/chosen": -307.5903015136719,
      "eval_logps/rejected": -309.0183410644531,
      "eval_loss": 0.49208447337150574,
      "eval_rewards/accuracies": 0.7460317611694336,
      "eval_rewards/chosen": -0.5635129809379578,
      "eval_rewards/margins": 1.3005269765853882,
      "eval_rewards/rejected": -1.8640400171279907,
      "eval_runtime": 431.8636,
      "eval_samples_per_second": 4.631,
      "eval_steps_per_second": 0.146,
      "step": 1100
    },
    {
      "epoch": 1.16,
      "learning_rate": 9.608463116858542e-08,
      "logits/chosen": -2.4096856117248535,
      "logits/rejected": -2.3269238471984863,
      "logps/chosen": -319.33123779296875,
      "logps/rejected": -314.37982177734375,
      "loss": 0.2832,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4104345738887787,
      "rewards/margins": 1.9642629623413086,
      "rewards/rejected": -2.37469744682312,
      "step": 1110
    },
    {
      "epoch": 1.17,
      "learning_rate": 9.15422581503224e-08,
      "logits/chosen": -2.4082019329071045,
      "logits/rejected": -2.3823094367980957,
      "logps/chosen": -281.8961181640625,
      "logps/rejected": -310.1343078613281,
      "loss": 0.2659,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4686800539493561,
      "rewards/margins": 1.9623409509658813,
      "rewards/rejected": -2.431021213531494,
      "step": 1120
    },
    {
      "epoch": 1.18,
      "learning_rate": 8.70856927946362e-08,
      "logits/chosen": -2.4076130390167236,
      "logits/rejected": -2.377281427383423,
      "logps/chosen": -313.17877197265625,
      "logps/rejected": -320.7528381347656,
      "loss": 0.2712,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3578185439109802,
      "rewards/margins": 1.8895679712295532,
      "rewards/rejected": -2.2473864555358887,
      "step": 1130
    },
    {
      "epoch": 1.19,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -2.4748058319091797,
      "logits/rejected": -2.3889963626861572,
      "logps/chosen": -317.75933837890625,
      "logps/rejected": -335.86553955078125,
      "loss": 0.2587,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.460686057806015,
      "rewards/margins": 1.9483623504638672,
      "rewards/rejected": -2.409048557281494,
      "step": 1140
    },
    {
      "epoch": 1.2,
      "learning_rate": 7.843959053281663e-08,
      "logits/chosen": -2.419320583343506,
      "logits/rejected": -2.3716225624084473,
      "logps/chosen": -308.89141845703125,
      "logps/rejected": -291.35797119140625,
      "loss": 0.3091,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3241550624370575,
      "rewards/margins": 1.9242489337921143,
      "rewards/rejected": -2.248404026031494,
      "step": 1150
    },
    {
      "epoch": 1.21,
      "learning_rate": 7.425473564358456e-08,
      "logits/chosen": -2.4155445098876953,
      "logits/rejected": -2.374995708465576,
      "logps/chosen": -285.75823974609375,
      "logps/rejected": -320.6084899902344,
      "loss": 0.2787,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.321144163608551,
      "rewards/margins": 2.070187568664551,
      "rewards/rejected": -2.391331911087036,
      "step": 1160
    },
    {
      "epoch": 1.22,
      "learning_rate": 7.016504991533726e-08,
      "logits/chosen": -2.4452130794525146,
      "logits/rejected": -2.416900157928467,
      "logps/chosen": -333.56890869140625,
      "logps/rejected": -288.9085998535156,
      "loss": 0.2819,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.34982940554618835,
      "rewards/margins": 1.8246349096298218,
      "rewards/rejected": -2.174464702606201,
      "step": 1170
    },
    {
      "epoch": 1.24,
      "learning_rate": 6.617274798504286e-08,
      "logits/chosen": -2.4035491943359375,
      "logits/rejected": -2.3629653453826904,
      "logps/chosen": -306.9756164550781,
      "logps/rejected": -289.35479736328125,
      "loss": 0.2805,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.36044904589653015,
      "rewards/margins": 1.8948723077774048,
      "rewards/rejected": -2.2553212642669678,
      "step": 1180
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.22799917546252e-08,
      "logits/chosen": -2.459360361099243,
      "logits/rejected": -2.391692638397217,
      "logps/chosen": -314.09185791015625,
      "logps/rejected": -300.3113098144531,
      "loss": 0.2546,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.302379310131073,
      "rewards/margins": 2.0588433742523193,
      "rewards/rejected": -2.361222743988037,
      "step": 1190
    },
    {
      "epoch": 1.26,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": -2.473045825958252,
      "logits/rejected": -2.456024408340454,
      "logps/chosen": -312.7765808105469,
      "logps/rejected": -322.79498291015625,
      "loss": 0.2932,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.290397971868515,
      "rewards/margins": 1.813227653503418,
      "rewards/rejected": -2.103625774383545,
      "step": 1200
    },
    {
      "epoch": 1.26,
      "eval_logits/chosen": -2.391538381576538,
      "eval_logits/rejected": -2.351095199584961,
      "eval_logps/chosen": -309.3631591796875,
      "eval_logps/rejected": -311.79766845703125,
      "eval_loss": 0.49240604043006897,
      "eval_rewards/accuracies": 0.7579365372657776,
      "eval_rewards/chosen": -0.6521540880203247,
      "eval_rewards/margins": 1.3508530855178833,
      "eval_rewards/rejected": -2.003007173538208,
      "eval_runtime": 414.0649,
      "eval_samples_per_second": 4.83,
      "eval_steps_per_second": 0.152,
      "step": 1200
    },
    {
      "epoch": 1.27,
      "learning_rate": 5.48014933308352e-08,
      "logits/chosen": -2.407109022140503,
      "logits/rejected": -2.3259758949279785,
      "logps/chosen": -230.8815155029297,
      "logps/rejected": -267.571044921875,
      "loss": 0.3043,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5537179112434387,
      "rewards/margins": 1.7985881567001343,
      "rewards/rejected": -2.3523058891296387,
      "step": 1210
    },
    {
      "epoch": 1.28,
      "learning_rate": 5.121980087628802e-08,
      "logits/chosen": -2.468632221221924,
      "logits/rejected": -2.407593250274658,
      "logps/chosen": -321.1299133300781,
      "logps/rejected": -319.76788330078125,
      "loss": 0.2968,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.3099203407764435,
      "rewards/margins": 2.04097580909729,
      "rewards/rejected": -2.350896120071411,
      "step": 1220
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.774575140626316e-08,
      "logits/chosen": -2.399498462677002,
      "logits/rejected": -2.317718982696533,
      "logps/chosen": -311.56378173828125,
      "logps/rejected": -322.6271057128906,
      "loss": 0.2885,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.34509938955307007,
      "rewards/margins": 1.8977705240249634,
      "rewards/rejected": -2.2428698539733887,
      "step": 1230
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.438122617983442e-08,
      "logits/chosen": -2.3652095794677734,
      "logits/rejected": -2.3344194889068604,
      "logps/chosen": -290.28790283203125,
      "logps/rejected": -294.1825866699219,
      "loss": 0.2989,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4549785256385803,
      "rewards/margins": 1.7224676609039307,
      "rewards/rejected": -2.177445888519287,
      "step": 1240
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.112804714676593e-08,
      "logits/chosen": -2.459439754486084,
      "logits/rejected": -2.4127402305603027,
      "logps/chosen": -292.1894226074219,
      "logps/rejected": -307.1498107910156,
      "loss": 0.2864,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.346891850233078,
      "rewards/margins": 1.9702911376953125,
      "rewards/rejected": -2.317183017730713,
      "step": 1250
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.798797596089351e-08,
      "logits/chosen": -2.427961587905884,
      "logits/rejected": -2.4093945026397705,
      "logps/chosen": -316.4217224121094,
      "logps/rejected": -341.4559631347656,
      "loss": 0.2918,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.42415857315063477,
      "rewards/margins": 1.8887336254119873,
      "rewards/rejected": -2.312892198562622,
      "step": 1260
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.496271302615869e-08,
      "logits/chosen": -2.4277498722076416,
      "logits/rejected": -2.35754656791687,
      "logps/chosen": -290.10235595703125,
      "logps/rejected": -304.0705261230469,
      "loss": 0.2807,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.38870421051979065,
      "rewards/margins": 2.0075557231903076,
      "rewards/rejected": -2.3962597846984863,
      "step": 1270
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.205389657580943e-08,
      "logits/chosen": -2.438321590423584,
      "logits/rejected": -2.3755178451538086,
      "logps/chosen": -283.269775390625,
      "logps/rejected": -337.3073425292969,
      "loss": 0.2519,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2695261836051941,
      "rewards/margins": 2.0451509952545166,
      "rewards/rejected": -2.3146772384643555,
      "step": 1280
    },
    {
      "epoch": 1.35,
      "learning_rate": 2.9263101785268252e-08,
      "logits/chosen": -2.4288270473480225,
      "logits/rejected": -2.39312481880188,
      "logps/chosen": -325.607421875,
      "logps/rejected": -317.9021911621094,
      "loss": 0.2867,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3596641421318054,
      "rewards/margins": 1.9093034267425537,
      "rewards/rejected": -2.268967390060425,
      "step": 1290
    },
    {
      "epoch": 1.36,
      "learning_rate": 2.659183991914696e-08,
      "logits/chosen": -2.419738531112671,
      "logits/rejected": -2.362929105758667,
      "logps/chosen": -296.9184875488281,
      "logps/rejected": -324.33587646484375,
      "loss": 0.275,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3492244482040405,
      "rewards/margins": 2.0286529064178467,
      "rewards/rejected": -2.3778772354125977,
      "step": 1300
    },
    {
      "epoch": 1.36,
      "eval_logits/chosen": -2.39336895942688,
      "eval_logits/rejected": -2.353144407272339,
      "eval_logps/chosen": -309.0525817871094,
      "eval_logps/rejected": -311.2369079589844,
      "eval_loss": 0.4915713965892792,
      "eval_rewards/accuracies": 0.7599206566810608,
      "eval_rewards/chosen": -0.6366247534751892,
      "eval_rewards/margins": 1.3383426666259766,
      "eval_rewards/rejected": -1.9749674797058105,
      "eval_runtime": 412.9249,
      "eval_samples_per_second": 4.843,
      "eval_steps_per_second": 0.153,
      "step": 1300
    },
    {
      "epoch": 1.37,
      "learning_rate": 2.4041557512869876e-08,
      "logits/chosen": -2.3756937980651855,
      "logits/rejected": -2.3104584217071533,
      "logps/chosen": -307.74395751953125,
      "logps/rejected": -316.4643859863281,
      "loss": 0.301,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.6529245972633362,
      "rewards/margins": 1.7977030277252197,
      "rewards/rejected": -2.4506278038024902,
      "step": 1310
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.1613635589349756e-08,
      "logits/chosen": -2.4178683757781982,
      "logits/rejected": -2.3683836460113525,
      "logps/chosen": -293.5295715332031,
      "logps/rejected": -329.43756103515625,
      "loss": 0.2802,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4704504907131195,
      "rewards/margins": 1.998552680015564,
      "rewards/rejected": -2.469003200531006,
      "step": 1320
    },
    {
      "epoch": 1.39,
      "learning_rate": 1.9309388911139424e-08,
      "logits/chosen": -2.4530060291290283,
      "logits/rejected": -2.417515993118286,
      "logps/chosen": -303.9961853027344,
      "logps/rejected": -316.8196716308594,
      "loss": 0.2883,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.4491092562675476,
      "rewards/margins": 1.8476078510284424,
      "rewards/rejected": -2.2967171669006348,
      "step": 1330
    },
    {
      "epoch": 1.4,
      "learning_rate": 1.713006526846439e-08,
      "logits/chosen": -2.3352913856506348,
      "logits/rejected": -2.3332459926605225,
      "logps/chosen": -309.6740417480469,
      "logps/rejected": -316.5276184082031,
      "loss": 0.3034,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.5785962343215942,
      "rewards/margins": 1.668442964553833,
      "rewards/rejected": -2.247039318084717,
      "step": 1340
    },
    {
      "epoch": 1.41,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": -2.444365978240967,
      "logits/rejected": -2.3748979568481445,
      "logps/chosen": -273.01177978515625,
      "logps/rejected": -300.568115234375,
      "loss": 0.2704,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5734509229660034,
      "rewards/margins": 1.8402652740478516,
      "rewards/rejected": -2.4137163162231445,
      "step": 1350
    },
    {
      "epoch": 1.42,
      "learning_rate": 1.3150839371417699e-08,
      "logits/chosen": -2.3913302421569824,
      "logits/rejected": -2.3794515132904053,
      "logps/chosen": -270.5587463378906,
      "logps/rejected": -277.6205749511719,
      "loss": 0.3003,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.3957940936088562,
      "rewards/margins": 1.739976167678833,
      "rewards/rejected": -2.135770320892334,
      "step": 1360
    },
    {
      "epoch": 1.43,
      "learning_rate": 1.1353091938067023e-08,
      "logits/chosen": -2.40122652053833,
      "logits/rejected": -2.317656993865967,
      "logps/chosen": -306.97454833984375,
      "logps/rejected": -288.46038818359375,
      "loss": 0.2884,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.4694311022758484,
      "rewards/margins": 1.7464059591293335,
      "rewards/rejected": -2.215837240219116,
      "step": 1370
    },
    {
      "epoch": 1.44,
      "learning_rate": 9.684576015420275e-09,
      "logits/chosen": -2.429955005645752,
      "logits/rejected": -2.340874433517456,
      "logps/chosen": -284.2897033691406,
      "logps/rejected": -346.4473571777344,
      "loss": 0.259,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.4956478178501129,
      "rewards/margins": 2.128859043121338,
      "rewards/rejected": -2.624507188796997,
      "step": 1380
    },
    {
      "epoch": 1.46,
      "learning_rate": 8.14619513428405e-09,
      "logits/chosen": -2.41424822807312,
      "logits/rejected": -2.4188144207000732,
      "logps/chosen": -284.09356689453125,
      "logps/rejected": -301.265380859375,
      "loss": 0.3023,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.6043499708175659,
      "rewards/margins": 1.725403070449829,
      "rewards/rejected": -2.3297529220581055,
      "step": 1390
    },
    {
      "epoch": 1.47,
      "learning_rate": 6.738782355044048e-09,
      "logits/chosen": -2.420943260192871,
      "logits/rejected": -2.3737356662750244,
      "logps/chosen": -295.2271423339844,
      "logps/rejected": -300.8685607910156,
      "loss": 0.2768,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.4684749245643616,
      "rewards/margins": 2.138996124267578,
      "rewards/rejected": -2.607471227645874,
      "step": 1400
    },
    {
      "epoch": 1.47,
      "eval_logits/chosen": -2.3908376693725586,
      "eval_logits/rejected": -2.3505396842956543,
      "eval_logps/chosen": -310.34185791015625,
      "eval_logps/rejected": -312.6646423339844,
      "eval_loss": 0.49215617775917053,
      "eval_rewards/accuracies": 0.7579365372657776,
      "eval_rewards/chosen": -0.7010902166366577,
      "eval_rewards/margins": 1.345264196395874,
      "eval_rewards/rejected": -2.0463547706604004,
      "eval_runtime": 434.5194,
      "eval_samples_per_second": 4.603,
      "eval_steps_per_second": 0.145,
      "step": 1400
    },
    {
      "epoch": 1.48,
      "learning_rate": 5.463099816548577e-09,
      "logits/chosen": -2.435065269470215,
      "logits/rejected": -2.368187427520752,
      "logps/chosen": -305.482666015625,
      "logps/rejected": -319.5362548828125,
      "loss": 0.2907,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.512501060962677,
      "rewards/margins": 1.8626611232757568,
      "rewards/rejected": -2.375162124633789,
      "step": 1410
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.319838323396691e-09,
      "logits/chosen": -2.425668239593506,
      "logits/rejected": -2.394260883331299,
      "logps/chosen": -285.38983154296875,
      "logps/rejected": -326.1392822265625,
      "loss": 0.28,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.43002137541770935,
      "rewards/margins": 1.9287302494049072,
      "rewards/rejected": -2.3587517738342285,
      "step": 1420
    },
    {
      "epoch": 1.5,
      "learning_rate": 3.309616971855195e-09,
      "logits/chosen": -2.3961329460144043,
      "logits/rejected": -2.3518338203430176,
      "logps/chosen": -313.55267333984375,
      "logps/rejected": -286.34320068359375,
      "loss": 0.2816,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.41422533988952637,
      "rewards/margins": 1.944323182106018,
      "rewards/rejected": -2.358548402786255,
      "step": 1430
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.4329828146074096e-09,
      "logits/chosen": -2.386317491531372,
      "logits/rejected": -2.3524794578552246,
      "logps/chosen": -313.0305480957031,
      "logps/rejected": -332.1868896484375,
      "loss": 0.2798,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3209957182407379,
      "rewards/margins": 2.147188663482666,
      "rewards/rejected": -2.46818470954895,
      "step": 1440
    },
    {
      "epoch": 1.52,
      "learning_rate": 1.690410564514244e-09,
      "logits/chosen": -2.411320209503174,
      "logits/rejected": -2.3845181465148926,
      "logps/chosen": -296.78228759765625,
      "logps/rejected": -306.8433837890625,
      "loss": 0.2898,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4139639735221863,
      "rewards/margins": 1.8331149816513062,
      "rewards/rejected": -2.2470791339874268,
      "step": 1450
    },
    {
      "epoch": 1.53,
      "learning_rate": 1.0823023375489126e-09,
      "logits/chosen": -2.468296766281128,
      "logits/rejected": -2.4058051109313965,
      "logps/chosen": -282.4747619628906,
      "logps/rejected": -294.89923095703125,
      "loss": 0.2817,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.3540436029434204,
      "rewards/margins": 1.8670495748519897,
      "rewards/rejected": -2.22109317779541,
      "step": 1460
    },
    {
      "epoch": 1.54,
      "learning_rate": 6.089874350439505e-10,
      "logits/chosen": -2.4278903007507324,
      "logits/rejected": -2.412611961364746,
      "logps/chosen": -309.91180419921875,
      "logps/rejected": -351.41607666015625,
      "loss": 0.2826,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.38388413190841675,
      "rewards/margins": 1.9360164403915405,
      "rewards/rejected": -2.3199005126953125,
      "step": 1470
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.707221653688585e-10,
      "logits/chosen": -2.3936755657196045,
      "logits/rejected": -2.3537259101867676,
      "logps/chosen": -316.6966247558594,
      "logps/rejected": -323.77154541015625,
      "loss": 0.2674,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3255850076675415,
      "rewards/margins": 2.003681182861328,
      "rewards/rejected": -2.329266309738159,
      "step": 1480
    },
    {
      "epoch": 1.56,
      "learning_rate": 6.768970513457151e-11,
      "logits/chosen": -2.3720383644104004,
      "logits/rejected": -2.346740245819092,
      "logps/chosen": -301.3364562988281,
      "logps/rejected": -312.578125,
      "loss": 0.2749,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.46938952803611755,
      "rewards/margins": 1.996596097946167,
      "rewards/rejected": -2.4659857749938965,
      "step": 1490
    },
    {
      "epoch": 1.57,
      "learning_rate": 0.0,
      "logits/chosen": -2.345372200012207,
      "logits/rejected": -2.3051817417144775,
      "logps/chosen": -275.92779541015625,
      "logps/rejected": -335.74761962890625,
      "loss": 0.2863,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.5277979969978333,
      "rewards/margins": 2.0437979698181152,
      "rewards/rejected": -2.5715959072113037,
      "step": 1500
    },
    {
      "epoch": 1.57,
      "eval_logits/chosen": -2.3900513648986816,
      "eval_logits/rejected": -2.3497886657714844,
      "eval_logps/chosen": -310.23736572265625,
      "eval_logps/rejected": -312.59942626953125,
      "eval_loss": 0.4916023015975952,
      "eval_rewards/accuracies": 0.7579365372657776,
      "eval_rewards/chosen": -0.6958636045455933,
      "eval_rewards/margins": 1.3472286462783813,
      "eval_rewards/rejected": -2.0430922508239746,
      "eval_runtime": 454.0528,
      "eval_samples_per_second": 4.405,
      "eval_steps_per_second": 0.139,
      "step": 1500
    },
    {
      "epoch": 1.57,
      "step": 1500,
      "total_flos": 0.0,
      "train_loss": 0.4390208276112874,
      "train_runtime": 42485.337,
      "train_samples_per_second": 2.26,
      "train_steps_per_second": 0.035
    }
  ],
  "logging_steps": 10,
  "max_steps": 1500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}