{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 100,
  "global_step": 4728,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.0570824524312896e-09,
      "logits/chosen": -2.491130828857422,
      "logits/rejected": -2.619523525238037,
      "logps/chosen": -252.33245849609375,
      "logps/rejected": -272.7114562988281,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.0570824524312896e-08,
      "logits/chosen": -2.4450111389160156,
      "logits/rejected": -2.593366861343384,
      "logps/chosen": -320.2229309082031,
      "logps/rejected": -342.5660400390625,
      "loss": 0.6957,
      "rewards/accuracies": 0.3611111044883728,
      "rewards/chosen": -0.0055116708390414715,
      "rewards/margins": -0.004262034315615892,
      "rewards/rejected": -0.0012496362905949354,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.114164904862579e-08,
      "logits/chosen": -2.548513412475586,
      "logits/rejected": -2.66241717338562,
      "logps/chosen": -246.2412109375,
      "logps/rejected": -265.8259582519531,
      "loss": 0.7003,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.005210032220929861,
      "rewards/margins": -0.013661216013133526,
      "rewards/rejected": 0.008451184257864952,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.1712473572938685e-08,
      "logits/chosen": -2.6386754512786865,
      "logits/rejected": -2.751687526702881,
      "logps/chosen": -227.59280395507812,
      "logps/rejected": -248.66867065429688,
      "loss": 0.6925,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.004014883656054735,
      "rewards/margins": 0.002194071654230356,
      "rewards/rejected": 0.0018208131659775972,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 4.228329809725158e-08,
      "logits/chosen": -2.5434064865112305,
      "logits/rejected": -2.6532297134399414,
      "logps/chosen": -295.59649658203125,
      "logps/rejected": -306.7847595214844,
      "loss": 0.6931,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.005541801452636719,
      "rewards/margins": 0.001035857480019331,
      "rewards/rejected": -0.006577658466994762,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 5.285412262156448e-08,
      "logits/chosen": -2.548645257949829,
      "logits/rejected": -2.686511516571045,
      "logps/chosen": -279.0761413574219,
      "logps/rejected": -332.60760498046875,
      "loss": 0.69,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0019467307720333338,
      "rewards/margins": 0.007388443686068058,
      "rewards/rejected": -0.005441713146865368,
      "step": 50
    },
    {
      "epoch": 0.04,
      "learning_rate": 6.342494714587737e-08,
      "logits/chosen": -2.6797890663146973,
      "logits/rejected": -2.811457395553589,
      "logps/chosen": -237.5199737548828,
      "logps/rejected": -262.17633056640625,
      "loss": 0.6938,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 0.005909358151257038,
      "rewards/margins": -0.00032917055068537593,
      "rewards/rejected": 0.006238528061658144,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 7.399577167019028e-08,
      "logits/chosen": -2.514037609100342,
      "logits/rejected": -2.657683849334717,
      "logps/chosen": -314.90362548828125,
      "logps/rejected": -322.99224853515625,
      "loss": 0.6955,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.004904889967292547,
      "rewards/margins": -0.003352522850036621,
      "rewards/rejected": -0.0015523673500865698,
      "step": 70
    },
    {
      "epoch": 0.05,
      "learning_rate": 8.456659619450317e-08,
      "logits/chosen": -2.5158894062042236,
      "logits/rejected": -2.6966891288757324,
      "logps/chosen": -249.71749877929688,
      "logps/rejected": -312.8682861328125,
      "loss": 0.6949,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.003046708181500435,
      "rewards/margins": -0.002077631652355194,
      "rewards/rejected": -0.0009690769948065281,
      "step": 80
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.513742071881606e-08,
      "logits/chosen": -2.518960475921631,
      "logits/rejected": -2.707387924194336,
      "logps/chosen": -218.50753784179688,
      "logps/rejected": -299.55718994140625,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006611490156501532,
      "rewards/margins": 0.01133851520717144,
      "rewards/rejected": -0.01795000582933426,
      "step": 90
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.0570824524312896e-07,
      "logits/chosen": -2.522345781326294,
      "logits/rejected": -2.6962521076202393,
      "logps/chosen": -228.46762084960938,
      "logps/rejected": -283.10614013671875,
      "loss": 0.6892,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.003819794859737158,
      "rewards/margins": 0.008738580159842968,
      "rewards/rejected": -0.004918784834444523,
      "step": 100
    },
    {
      "epoch": 0.06,
      "eval_logits/chosen": -2.6576220989227295,
      "eval_logits/rejected": -2.79404616355896,
      "eval_logps/chosen": -270.919921875,
      "eval_logps/rejected": -286.7756652832031,
      "eval_loss": 0.6904468536376953,
      "eval_rewards/accuracies": 0.4692307710647583,
      "eval_rewards/chosen": -0.0007067565456964076,
      "eval_rewards/margins": 0.006087994668632746,
      "eval_rewards/rejected": -0.006794750690460205,
      "eval_runtime": 103.8445,
      "eval_samples_per_second": 2.484,
      "eval_steps_per_second": 0.626,
      "step": 100
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.1627906976744186e-07,
      "logits/chosen": -2.551213264465332,
      "logits/rejected": -2.698561906814575,
      "logps/chosen": -248.2393035888672,
      "logps/rejected": -288.84503173828125,
      "loss": 0.6936,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.0040052225813269615,
      "rewards/margins": 1.28507617773721e-05,
      "rewards/rejected": -0.004018072970211506,
      "step": 110
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.2684989429175474e-07,
      "logits/chosen": -2.6813459396362305,
      "logits/rejected": -2.7697396278381348,
      "logps/chosen": -259.8791198730469,
      "logps/rejected": -253.40066528320312,
      "loss": 0.6897,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0011179109569638968,
      "rewards/margins": 0.007453221827745438,
      "rewards/rejected": -0.008571133017539978,
      "step": 120
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.3742071881606765e-07,
      "logits/chosen": -2.5724005699157715,
      "logits/rejected": -2.6868607997894287,
      "logps/chosen": -244.68881225585938,
      "logps/rejected": -283.0400390625,
      "loss": 0.6855,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.005307340994477272,
      "rewards/margins": 0.01705842837691307,
      "rewards/rejected": -0.022365763783454895,
      "step": 130
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.4799154334038056e-07,
      "logits/chosen": -2.6790432929992676,
      "logits/rejected": -2.77896785736084,
      "logps/chosen": -286.0615234375,
      "logps/rejected": -282.4619140625,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009276931174099445,
      "rewards/margins": 0.004257139749825001,
      "rewards/rejected": -0.01353407185524702,
      "step": 140
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.5856236786469342e-07,
      "logits/chosen": -2.5402438640594482,
      "logits/rejected": -2.6776623725891113,
      "logps/chosen": -262.10064697265625,
      "logps/rejected": -303.8569641113281,
      "loss": 0.6865,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.005709734279662371,
      "rewards/margins": 0.014272456057369709,
      "rewards/rejected": -0.019982188940048218,
      "step": 150
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.6913319238900633e-07,
      "logits/chosen": -2.5357205867767334,
      "logits/rejected": -2.6745407581329346,
      "logps/chosen": -254.10195922851562,
      "logps/rejected": -288.9562072753906,
      "loss": 0.6838,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.011983389966189861,
      "rewards/margins": 0.019775032997131348,
      "rewards/rejected": -0.031758420169353485,
      "step": 160
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.7970401691331924e-07,
      "logits/chosen": -2.5182926654815674,
      "logits/rejected": -2.659651041030884,
      "logps/chosen": -281.615478515625,
      "logps/rejected": -332.77020263671875,
      "loss": 0.6864,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.014197200536727905,
      "rewards/margins": 0.014835876412689686,
      "rewards/rejected": -0.029033079743385315,
      "step": 170
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.9027484143763213e-07,
      "logits/chosen": -2.6197454929351807,
      "logits/rejected": -2.745800733566284,
      "logps/chosen": -255.544677734375,
      "logps/rejected": -280.203857421875,
      "loss": 0.6825,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.007357417140156031,
      "rewards/margins": 0.02221287600696087,
      "rewards/rejected": -0.014855461195111275,
      "step": 180
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.00845665961945e-07,
      "logits/chosen": -2.527393102645874,
      "logits/rejected": -2.6684184074401855,
      "logps/chosen": -211.38095092773438,
      "logps/rejected": -290.3387145996094,
      "loss": 0.6834,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.006614250130951405,
      "rewards/margins": 0.0201758723706007,
      "rewards/rejected": -0.02679012343287468,
      "step": 190
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.1141649048625792e-07,
      "logits/chosen": -2.6162755489349365,
      "logits/rejected": -2.752476692199707,
      "logps/chosen": -245.95785522460938,
      "logps/rejected": -286.5357971191406,
      "loss": 0.6767,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.01054263673722744,
      "rewards/margins": 0.03406267613172531,
      "rewards/rejected": -0.0446053110063076,
      "step": 200
    },
    {
      "epoch": 0.13,
      "eval_logits/chosen": -2.656838893890381,
      "eval_logits/rejected": -2.7931315898895264,
      "eval_logps/chosen": -270.972412109375,
      "eval_logps/rejected": -287.13726806640625,
      "eval_loss": 0.6754144430160522,
      "eval_rewards/accuracies": 0.6384615302085876,
      "eval_rewards/chosen": -0.00595523277297616,
      "eval_rewards/margins": 0.03699635714292526,
      "eval_rewards/rejected": -0.042951587587594986,
      "eval_runtime": 102.435,
      "eval_samples_per_second": 2.519,
      "eval_steps_per_second": 0.635,
      "step": 200
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.219873150105708e-07,
      "logits/chosen": -2.542800188064575,
      "logits/rejected": -2.6776413917541504,
      "logps/chosen": -223.8124542236328,
      "logps/rejected": -274.4298400878906,
      "loss": 0.6773,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.010664287023246288,
      "rewards/margins": 0.033305078744888306,
      "rewards/rejected": -0.04396936297416687,
      "step": 210
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.3255813953488372e-07,
      "logits/chosen": -2.4896938800811768,
      "logits/rejected": -2.6893773078918457,
      "logps/chosen": -267.34185791015625,
      "logps/rejected": -316.0325622558594,
      "loss": 0.6635,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0003847453626804054,
      "rewards/margins": 0.061600469052791595,
      "rewards/rejected": -0.06198521703481674,
      "step": 220
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.431289640591966e-07,
      "logits/chosen": -2.5686957836151123,
      "logits/rejected": -2.7455496788024902,
      "logps/chosen": -235.91043090820312,
      "logps/rejected": -291.48785400390625,
      "loss": 0.6743,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.015519057400524616,
      "rewards/margins": 0.03921210393309593,
      "rewards/rejected": -0.054731160402297974,
      "step": 230
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.536997885835095e-07,
      "logits/chosen": -2.7444443702697754,
      "logits/rejected": -2.7797598838806152,
      "logps/chosen": -311.966064453125,
      "logps/rejected": -302.85906982421875,
      "loss": 0.6731,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.004274768754839897,
      "rewards/margins": 0.042329687625169754,
      "rewards/rejected": -0.0466044545173645,
      "step": 240
    },
    {
      "epoch": 0.16,
      "learning_rate": 2.642706131078224e-07,
      "logits/chosen": -2.4815051555633545,
      "logits/rejected": -2.6796867847442627,
      "logps/chosen": -232.5732879638672,
      "logps/rejected": -280.8894348144531,
      "loss": 0.6603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007849951274693012,
      "rewards/margins": 0.06879496574401855,
      "rewards/rejected": -0.07664491981267929,
      "step": 250
    },
    {
      "epoch": 0.16,
      "learning_rate": 2.748414376321353e-07,
      "logits/chosen": -2.611215829849243,
      "logits/rejected": -2.712291717529297,
      "logps/chosen": -256.3820495605469,
      "logps/rejected": -280.9920349121094,
      "loss": 0.6607,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.006520071066915989,
      "rewards/margins": 0.06718841940164566,
      "rewards/rejected": -0.07370848953723907,
      "step": 260
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.854122621564482e-07,
      "logits/chosen": -2.624483346939087,
      "logits/rejected": -2.7304062843322754,
      "logps/chosen": -280.4382629394531,
      "logps/rejected": -300.1388854980469,
      "loss": 0.671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.024707507342100143,
      "rewards/margins": 0.046161822974681854,
      "rewards/rejected": -0.0708693265914917,
      "step": 270
    },
    {
      "epoch": 0.18,
      "learning_rate": 2.959830866807611e-07,
      "logits/chosen": -2.597618818283081,
      "logits/rejected": -2.729827404022217,
      "logps/chosen": -258.9872741699219,
      "logps/rejected": -284.0447692871094,
      "loss": 0.6536,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.016296977177262306,
      "rewards/margins": 0.08284956216812134,
      "rewards/rejected": -0.0991465374827385,
      "step": 280
    },
    {
      "epoch": 0.18,
      "learning_rate": 3.0655391120507393e-07,
      "logits/chosen": -2.544942855834961,
      "logits/rejected": -2.7117061614990234,
      "logps/chosen": -275.4362487792969,
      "logps/rejected": -311.9210510253906,
      "loss": 0.638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.012941551394760609,
      "rewards/margins": 0.1152975782752037,
      "rewards/rejected": -0.12823912501335144,
      "step": 290
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.1712473572938684e-07,
      "logits/chosen": -2.5376856327056885,
      "logits/rejected": -2.653120994567871,
      "logps/chosen": -217.09286499023438,
      "logps/rejected": -251.13394165039062,
      "loss": 0.6493,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.022592002525925636,
      "rewards/margins": 0.09205517917871475,
      "rewards/rejected": -0.11464717239141464,
      "step": 300
    },
    {
      "epoch": 0.19,
      "eval_logits/chosen": -2.6560781002044678,
      "eval_logits/rejected": -2.79219651222229,
      "eval_logps/chosen": -271.01739501953125,
      "eval_logps/rejected": -287.8587646484375,
      "eval_loss": 0.6430513858795166,
      "eval_rewards/accuracies": 0.7884615659713745,
      "eval_rewards/chosen": -0.010456576012074947,
      "eval_rewards/margins": 0.10464820265769958,
      "eval_rewards/rejected": -0.1151047796010971,
      "eval_runtime": 101.5222,
      "eval_samples_per_second": 2.541,
      "eval_steps_per_second": 0.64,
      "step": 300
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.2769556025369975e-07,
      "logits/chosen": -2.563730239868164,
      "logits/rejected": -2.7113027572631836,
      "logps/chosen": -225.3623504638672,
      "logps/rejected": -264.43841552734375,
      "loss": 0.6363,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.008831793442368507,
      "rewards/margins": 0.1197088211774826,
      "rewards/rejected": -0.12854060530662537,
      "step": 310
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.3826638477801266e-07,
      "logits/chosen": -2.519477128982544,
      "logits/rejected": -2.692214250564575,
      "logps/chosen": -280.89764404296875,
      "logps/rejected": -322.5290832519531,
      "loss": 0.6276,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.02033248357474804,
      "rewards/margins": 0.13884124159812927,
      "rewards/rejected": -0.15917374193668365,
      "step": 320
    },
    {
      "epoch": 0.21,
      "learning_rate": 3.4883720930232557e-07,
      "logits/chosen": -2.563596725463867,
      "logits/rejected": -2.708038806915283,
      "logps/chosen": -268.1693420410156,
      "logps/rejected": -304.6720886230469,
      "loss": 0.6276,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.02589145302772522,
      "rewards/margins": 0.13810984790325165,
      "rewards/rejected": -0.16400131583213806,
      "step": 330
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.594080338266385e-07,
      "logits/chosen": -2.679243564605713,
      "logits/rejected": -2.7936196327209473,
      "logps/chosen": -256.6110534667969,
      "logps/rejected": -298.4222106933594,
      "loss": 0.6284,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.005742775276303291,
      "rewards/margins": 0.13771875202655792,
      "rewards/rejected": -0.14346154034137726,
      "step": 340
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.699788583509514e-07,
      "logits/chosen": -2.522581100463867,
      "logits/rejected": -2.6692380905151367,
      "logps/chosen": -251.6295623779297,
      "logps/rejected": -285.0303955078125,
      "loss": 0.6207,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.02315356209874153,
      "rewards/margins": 0.15359243750572205,
      "rewards/rejected": -0.17674598097801208,
      "step": 350
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.8054968287526425e-07,
      "logits/chosen": -2.614532709121704,
      "logits/rejected": -2.748246431350708,
      "logps/chosen": -301.9349670410156,
      "logps/rejected": -317.8115539550781,
      "loss": 0.6126,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.013967705890536308,
      "rewards/margins": 0.17277029156684875,
      "rewards/rejected": -0.18673798441886902,
      "step": 360
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.9112050739957716e-07,
      "logits/chosen": -2.4634742736816406,
      "logits/rejected": -2.5858407020568848,
      "logps/chosen": -280.56011962890625,
      "logps/rejected": -306.52374267578125,
      "loss": 0.6078,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03250822052359581,
      "rewards/margins": 0.18415920436382294,
      "rewards/rejected": -0.21666744351387024,
      "step": 370
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.0169133192389e-07,
      "logits/chosen": -2.561014175415039,
      "logits/rejected": -2.7447683811187744,
      "logps/chosen": -251.95956420898438,
      "logps/rejected": -296.85308837890625,
      "loss": 0.6027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.029534289613366127,
      "rewards/margins": 0.19565005600452423,
      "rewards/rejected": -0.22518432140350342,
      "step": 380
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1226215644820293e-07,
      "logits/chosen": -2.5677709579467773,
      "logits/rejected": -2.714022159576416,
      "logps/chosen": -275.03216552734375,
      "logps/rejected": -334.52850341796875,
      "loss": 0.5893,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.03614232689142227,
      "rewards/margins": 0.22716024518013,
      "rewards/rejected": -0.2633025646209717,
      "step": 390
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.2283298097251584e-07,
      "logits/chosen": -2.687054395675659,
      "logits/rejected": -2.8174469470977783,
      "logps/chosen": -256.0891418457031,
      "logps/rejected": -291.55743408203125,
      "loss": 0.5809,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.033122412860393524,
      "rewards/margins": 0.2465277463197708,
      "rewards/rejected": -0.27965015172958374,
      "step": 400
    },
    {
      "epoch": 0.25,
      "eval_logits/chosen": -2.653402090072632,
      "eval_logits/rejected": -2.7892954349517822,
      "eval_logps/chosen": -271.2578125,
      "eval_logps/rejected": -289.3571472167969,
      "eval_loss": 0.587898313999176,
      "eval_rewards/accuracies": 0.8307692408561707,
      "eval_rewards/chosen": -0.03449693322181702,
      "eval_rewards/margins": 0.2304457128047943,
      "eval_rewards/rejected": -0.2649426758289337,
      "eval_runtime": 107.1518,
      "eval_samples_per_second": 2.408,
      "eval_steps_per_second": 0.607,
      "step": 400
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.3340380549682875e-07,
      "logits/chosen": -2.4630043506622314,
      "logits/rejected": -2.627161979675293,
      "logps/chosen": -194.3063507080078,
      "logps/rejected": -256.1365966796875,
      "loss": 0.5744,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.032422542572021484,
      "rewards/margins": 0.26237785816192627,
      "rewards/rejected": -0.29480037093162537,
      "step": 410
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.439746300211416e-07,
      "logits/chosen": -2.6319987773895264,
      "logits/rejected": -2.7381770610809326,
      "logps/chosen": -243.7211456298828,
      "logps/rejected": -282.017333984375,
      "loss": 0.5886,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05108586698770523,
      "rewards/margins": 0.2314957082271576,
      "rewards/rejected": -0.2825815677642822,
      "step": 420
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -2.549373149871826,
      "logits/rejected": -2.697718620300293,
      "logps/chosen": -238.696533203125,
      "logps/rejected": -296.9385070800781,
      "loss": 0.5739,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.03591006249189377,
      "rewards/margins": 0.26528820395469666,
      "rewards/rejected": -0.301198273897171,
      "step": 430
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.6511627906976743e-07,
      "logits/chosen": -2.491665840148926,
      "logits/rejected": -2.6627883911132812,
      "logps/chosen": -191.23208618164062,
      "logps/rejected": -264.14703369140625,
      "loss": 0.5447,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.043362896889448166,
      "rewards/margins": 0.33499208092689514,
      "rewards/rejected": -0.3783549964427948,
      "step": 440
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.7568710359408034e-07,
      "logits/chosen": -2.4616281986236572,
      "logits/rejected": -2.5903401374816895,
      "logps/chosen": -258.3981018066406,
      "logps/rejected": -291.92340087890625,
      "loss": 0.5502,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.04792201891541481,
      "rewards/margins": 0.325509250164032,
      "rewards/rejected": -0.3734312653541565,
      "step": 450
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.862579281183933e-07,
      "logits/chosen": -2.586905002593994,
      "logits/rejected": -2.6746628284454346,
      "logps/chosen": -266.5275573730469,
      "logps/rejected": -284.2345886230469,
      "loss": 0.5242,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.060103993862867355,
      "rewards/margins": 0.3883165717124939,
      "rewards/rejected": -0.44842061400413513,
      "step": 460
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.968287526427061e-07,
      "logits/chosen": -2.5683646202087402,
      "logits/rejected": -2.699049234390259,
      "logps/chosen": -190.07839965820312,
      "logps/rejected": -245.98519897460938,
      "loss": 0.5348,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.04075492545962334,
      "rewards/margins": 0.3719077408313751,
      "rewards/rejected": -0.41266268491744995,
      "step": 470
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.991774383078731e-07,
      "logits/chosen": -2.650360584259033,
      "logits/rejected": -2.7334964275360107,
      "logps/chosen": -281.48797607421875,
      "logps/rejected": -287.2583923339844,
      "loss": 0.5184,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.05656028911471367,
      "rewards/margins": 0.4161533713340759,
      "rewards/rejected": -0.4727136492729187,
      "step": 480
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.980023501762632e-07,
      "logits/chosen": -2.5589873790740967,
      "logits/rejected": -2.648350238800049,
      "logps/chosen": -247.8079376220703,
      "logps/rejected": -274.936767578125,
      "loss": 0.5235,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07316668331623077,
      "rewards/margins": 0.40529927611351013,
      "rewards/rejected": -0.4784659445285797,
      "step": 490
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.968272620446533e-07,
      "logits/chosen": -2.597858428955078,
      "logits/rejected": -2.705444574356079,
      "logps/chosen": -262.1869812011719,
      "logps/rejected": -277.0486755371094,
      "loss": 0.4994,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.09247130900621414,
      "rewards/margins": 0.4590112566947937,
      "rewards/rejected": -0.551482617855072,
      "step": 500
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -2.649930000305176,
      "eval_logits/rejected": -2.7851059436798096,
      "eval_logps/chosen": -271.687255859375,
      "eval_logps/rejected": -292.00421142578125,
      "eval_loss": 0.5042799115180969,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.07744207978248596,
      "eval_rewards/margins": 0.45220136642456055,
      "eval_rewards/rejected": -0.5296434164047241,
      "eval_runtime": 103.7438,
      "eval_samples_per_second": 2.487,
      "eval_steps_per_second": 0.627,
      "step": 500
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.956521739130435e-07,
      "logits/chosen": -2.606553554534912,
      "logits/rejected": -2.696528911590576,
      "logps/chosen": -284.43707275390625,
      "logps/rejected": -296.597900390625,
      "loss": 0.5132,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08263508975505829,
      "rewards/margins": 0.43400269746780396,
      "rewards/rejected": -0.5166377425193787,
      "step": 510
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.944770857814336e-07,
      "logits/chosen": -2.5119898319244385,
      "logits/rejected": -2.6895546913146973,
      "logps/chosen": -247.837890625,
      "logps/rejected": -332.5614318847656,
      "loss": 0.4892,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0914909616112709,
      "rewards/margins": 0.4965516924858093,
      "rewards/rejected": -0.588042676448822,
      "step": 520
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.933019976498237e-07,
      "logits/chosen": -2.4336352348327637,
      "logits/rejected": -2.5868098735809326,
      "logps/chosen": -269.35888671875,
      "logps/rejected": -327.2439880371094,
      "loss": 0.4712,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07643160223960876,
      "rewards/margins": 0.5561062693595886,
      "rewards/rejected": -0.632537841796875,
      "step": 530
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.921269095182138e-07,
      "logits/chosen": -2.5371692180633545,
      "logits/rejected": -2.6853349208831787,
      "logps/chosen": -231.96945190429688,
      "logps/rejected": -296.11871337890625,
      "loss": 0.4707,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.038751743733882904,
      "rewards/margins": 0.5505437254905701,
      "rewards/rejected": -0.589295506477356,
      "step": 540
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.909518213866039e-07,
      "logits/chosen": -2.5543642044067383,
      "logits/rejected": -2.7306723594665527,
      "logps/chosen": -251.3912353515625,
      "logps/rejected": -305.6792297363281,
      "loss": 0.4609,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08980365097522736,
      "rewards/margins": 0.5866891741752625,
      "rewards/rejected": -0.6764928102493286,
      "step": 550
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.897767332549941e-07,
      "logits/chosen": -2.6200075149536133,
      "logits/rejected": -2.7147746086120605,
      "logps/chosen": -319.3304748535156,
      "logps/rejected": -326.9119567871094,
      "loss": 0.4488,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07888631522655487,
      "rewards/margins": 0.6244070529937744,
      "rewards/rejected": -0.7032934427261353,
      "step": 560
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.886016451233842e-07,
      "logits/chosen": -2.5328903198242188,
      "logits/rejected": -2.6710097789764404,
      "logps/chosen": -237.8115997314453,
      "logps/rejected": -267.71307373046875,
      "loss": 0.4684,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.12399999052286148,
      "rewards/margins": 0.5662329792976379,
      "rewards/rejected": -0.6902328729629517,
      "step": 570
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.874265569917743e-07,
      "logits/chosen": -2.526376962661743,
      "logits/rejected": -2.659062623977661,
      "logps/chosen": -228.30264282226562,
      "logps/rejected": -280.83441162109375,
      "loss": 0.4363,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.10060425102710724,
      "rewards/margins": 0.6609746217727661,
      "rewards/rejected": -0.7615788578987122,
      "step": 580
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.862514688601645e-07,
      "logits/chosen": -2.558803081512451,
      "logits/rejected": -2.656737804412842,
      "logps/chosen": -299.07330322265625,
      "logps/rejected": -309.98504638671875,
      "loss": 0.4375,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0838431715965271,
      "rewards/margins": 0.6654406189918518,
      "rewards/rejected": -0.7492837905883789,
      "step": 590
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.850763807285546e-07,
      "logits/chosen": -2.528730630874634,
      "logits/rejected": -2.6592578887939453,
      "logps/chosen": -221.7564697265625,
      "logps/rejected": -271.0441589355469,
      "loss": 0.4093,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1245318055152893,
      "rewards/margins": 0.7670997381210327,
      "rewards/rejected": -0.8916316032409668,
      "step": 600
    },
    {
      "epoch": 0.38,
      "eval_logits/chosen": -2.6476452350616455,
      "eval_logits/rejected": -2.7820308208465576,
      "eval_logps/chosen": -272.17999267578125,
      "eval_logps/rejected": -294.75042724609375,
      "eval_loss": 0.4359889030456543,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.12671422958374023,
      "eval_rewards/margins": 0.6775518655776978,
      "eval_rewards/rejected": -0.8042660355567932,
      "eval_runtime": 103.6189,
      "eval_samples_per_second": 2.49,
      "eval_steps_per_second": 0.627,
      "step": 600
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.839012925969447e-07,
      "logits/chosen": -2.4936811923980713,
      "logits/rejected": -2.6524298191070557,
      "logps/chosen": -260.4691162109375,
      "logps/rejected": -306.52362060546875,
      "loss": 0.4026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1442100554704666,
      "rewards/margins": 0.7929333448410034,
      "rewards/rejected": -0.9371433258056641,
      "step": 610
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.827262044653348e-07,
      "logits/chosen": -2.4566304683685303,
      "logits/rejected": -2.5852420330047607,
      "logps/chosen": -298.7530212402344,
      "logps/rejected": -320.64581298828125,
      "loss": 0.4131,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.14908508956432343,
      "rewards/margins": 0.7372461557388306,
      "rewards/rejected": -0.8863312602043152,
      "step": 620
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.81551116333725e-07,
      "logits/chosen": -2.5423336029052734,
      "logits/rejected": -2.6401870250701904,
      "logps/chosen": -261.72125244140625,
      "logps/rejected": -294.8016662597656,
      "loss": 0.4091,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13849499821662903,
      "rewards/margins": 0.7663065195083618,
      "rewards/rejected": -0.9048014879226685,
      "step": 630
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.803760282021151e-07,
      "logits/chosen": -2.6755101680755615,
      "logits/rejected": -2.792830467224121,
      "logps/chosen": -259.3096923828125,
      "logps/rejected": -275.906982421875,
      "loss": 0.4528,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.15274348855018616,
      "rewards/margins": 0.637208104133606,
      "rewards/rejected": -0.7899516224861145,
      "step": 640
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.792009400705052e-07,
      "logits/chosen": -2.533348560333252,
      "logits/rejected": -2.708235025405884,
      "logps/chosen": -225.9936981201172,
      "logps/rejected": -297.2400817871094,
      "loss": 0.3984,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.16008391976356506,
      "rewards/margins": 0.8172070384025574,
      "rewards/rejected": -0.9772909283638,
      "step": 650
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.780258519388953e-07,
      "logits/chosen": -2.587038516998291,
      "logits/rejected": -2.7104010581970215,
      "logps/chosen": -230.0919952392578,
      "logps/rejected": -271.63787841796875,
      "loss": 0.4178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18881990015506744,
      "rewards/margins": 0.7464120984077454,
      "rewards/rejected": -0.9352320432662964,
      "step": 660
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.768507638072856e-07,
      "logits/chosen": -2.5364294052124023,
      "logits/rejected": -2.6631999015808105,
      "logps/chosen": -307.46295166015625,
      "logps/rejected": -325.50567626953125,
      "loss": 0.3805,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.1762232780456543,
      "rewards/margins": 0.878200888633728,
      "rewards/rejected": -1.0544241666793823,
      "step": 670
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.7567567567567566e-07,
      "logits/chosen": -2.6659674644470215,
      "logits/rejected": -2.7927112579345703,
      "logps/chosen": -235.7435302734375,
      "logps/rejected": -287.24395751953125,
      "loss": 0.4263,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11412607133388519,
      "rewards/margins": 0.7451529502868652,
      "rewards/rejected": -0.8592789769172668,
      "step": 680
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.745005875440658e-07,
      "logits/chosen": -2.6435635089874268,
      "logits/rejected": -2.766162633895874,
      "logps/chosen": -219.01107788085938,
      "logps/rejected": -269.82464599609375,
      "loss": 0.4009,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.14512372016906738,
      "rewards/margins": 0.8333772420883179,
      "rewards/rejected": -0.97850102186203,
      "step": 690
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.733254994124559e-07,
      "logits/chosen": -2.5526251792907715,
      "logits/rejected": -2.683788537979126,
      "logps/chosen": -269.64984130859375,
      "logps/rejected": -308.91290283203125,
      "loss": 0.3951,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.12358339875936508,
      "rewards/margins": 0.8354290723800659,
      "rewards/rejected": -0.959012508392334,
      "step": 700
    },
    {
      "epoch": 0.44,
      "eval_logits/chosen": -2.6458818912506104,
      "eval_logits/rejected": -2.779630184173584,
      "eval_logps/chosen": -272.6434326171875,
      "eval_logps/rejected": -297.30792236328125,
      "eval_loss": 0.38438332080841064,
      "eval_rewards/accuracies": 0.8423076868057251,
      "eval_rewards/chosen": -0.1730610877275467,
      "eval_rewards/margins": 0.8869585394859314,
      "eval_rewards/rejected": -1.0600197315216064,
      "eval_runtime": 103.4145,
      "eval_samples_per_second": 2.495,
      "eval_steps_per_second": 0.629,
      "step": 700
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.72150411280846e-07,
      "logits/chosen": -2.4890854358673096,
      "logits/rejected": -2.653275728225708,
      "logps/chosen": -275.90692138671875,
      "logps/rejected": -314.3439636230469,
      "loss": 0.3279,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.15251557528972626,
      "rewards/margins": 1.096553087234497,
      "rewards/rejected": -1.2490684986114502,
      "step": 710
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.7097532314923617e-07,
      "logits/chosen": -2.5283074378967285,
      "logits/rejected": -2.6887707710266113,
      "logps/chosen": -227.91616821289062,
      "logps/rejected": -293.15692138671875,
      "loss": 0.3894,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.17320795357227325,
      "rewards/margins": 0.8666555285453796,
      "rewards/rejected": -1.0398635864257812,
      "step": 720
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.6980023501762627e-07,
      "logits/chosen": -2.5899016857147217,
      "logits/rejected": -2.669069528579712,
      "logps/chosen": -281.13323974609375,
      "logps/rejected": -308.6754150390625,
      "loss": 0.3568,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.14439600706100464,
      "rewards/margins": 0.9995133280754089,
      "rewards/rejected": -1.1439093351364136,
      "step": 730
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.686251468860165e-07,
      "logits/chosen": -2.525472402572632,
      "logits/rejected": -2.644094467163086,
      "logps/chosen": -250.12802124023438,
      "logps/rejected": -288.5372009277344,
      "loss": 0.3798,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1782929003238678,
      "rewards/margins": 0.9057542681694031,
      "rewards/rejected": -1.0840469598770142,
      "step": 740
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.674500587544066e-07,
      "logits/chosen": -2.5681750774383545,
      "logits/rejected": -2.680637836456299,
      "logps/chosen": -243.38369750976562,
      "logps/rejected": -293.89654541015625,
      "loss": 0.3697,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.15879176557064056,
      "rewards/margins": 0.9550620317459106,
      "rewards/rejected": -1.1138536930084229,
      "step": 750
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.662749706227967e-07,
      "logits/chosen": -2.631460666656494,
      "logits/rejected": -2.796536922454834,
      "logps/chosen": -283.63787841796875,
      "logps/rejected": -327.7880554199219,
      "loss": 0.3438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1840469092130661,
      "rewards/margins": 1.0925601720809937,
      "rewards/rejected": -1.2766070365905762,
      "step": 760
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.6509988249118683e-07,
      "logits/chosen": -2.516369104385376,
      "logits/rejected": -2.684140205383301,
      "logps/chosen": -227.29092407226562,
      "logps/rejected": -281.3671569824219,
      "loss": 0.3274,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.20482750236988068,
      "rewards/margins": 1.1262274980545044,
      "rewards/rejected": -1.3310550451278687,
      "step": 770
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.6392479435957693e-07,
      "logits/chosen": -2.6184399127960205,
      "logits/rejected": -2.7169413566589355,
      "logps/chosen": -260.841064453125,
      "logps/rejected": -285.57061767578125,
      "loss": 0.3517,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.19830329716205597,
      "rewards/margins": 1.0601216554641724,
      "rewards/rejected": -1.258424997329712,
      "step": 780
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.6274970622796704e-07,
      "logits/chosen": -2.4811275005340576,
      "logits/rejected": -2.662588357925415,
      "logps/chosen": -223.5006103515625,
      "logps/rejected": -307.238037109375,
      "loss": 0.3096,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.24571017920970917,
      "rewards/margins": 1.1898123025894165,
      "rewards/rejected": -1.435522437095642,
      "step": 790
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.6157461809635724e-07,
      "logits/chosen": -2.5960936546325684,
      "logits/rejected": -2.7115960121154785,
      "logps/chosen": -297.3627014160156,
      "logps/rejected": -316.33795166015625,
      "loss": 0.3307,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.23634369671344757,
      "rewards/margins": 1.108078956604004,
      "rewards/rejected": -1.344422698020935,
      "step": 800
    },
    {
      "epoch": 0.51,
      "eval_logits/chosen": -2.6434166431427,
      "eval_logits/rejected": -2.776407480239868,
      "eval_logps/chosen": -273.1207580566406,
      "eval_logps/rejected": -299.9596862792969,
      "eval_loss": 0.3412703573703766,
      "eval_rewards/accuracies": 0.8346154093742371,
      "eval_rewards/chosen": -0.22079385817050934,
      "eval_rewards/margins": 1.1043978929519653,
      "eval_rewards/rejected": -1.325191617012024,
      "eval_runtime": 108.2168,
      "eval_samples_per_second": 2.384,
      "eval_steps_per_second": 0.601,
      "step": 800
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.6039952996474734e-07,
      "logits/chosen": -2.5736896991729736,
      "logits/rejected": -2.6705596446990967,
      "logps/chosen": -264.46075439453125,
      "logps/rejected": -286.5223083496094,
      "loss": 0.3646,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2299632728099823,
      "rewards/margins": 1.0295650959014893,
      "rewards/rejected": -1.2595282793045044,
      "step": 810
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.5922444183313745e-07,
      "logits/chosen": -2.557309627532959,
      "logits/rejected": -2.6736693382263184,
      "logps/chosen": -262.67535400390625,
      "logps/rejected": -295.8022766113281,
      "loss": 0.3392,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.23060734570026398,
      "rewards/margins": 1.1270842552185059,
      "rewards/rejected": -1.357691764831543,
      "step": 820
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.580493537015276e-07,
      "logits/chosen": -2.5337347984313965,
      "logits/rejected": -2.66684627532959,
      "logps/chosen": -281.9819030761719,
      "logps/rejected": -312.3790588378906,
      "loss": 0.3207,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.29394394159317017,
      "rewards/margins": 1.1960766315460205,
      "rewards/rejected": -1.490020513534546,
      "step": 830
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.568742655699177e-07,
      "logits/chosen": -2.580623149871826,
      "logits/rejected": -2.687054395675659,
      "logps/chosen": -276.7775573730469,
      "logps/rejected": -307.50848388671875,
      "loss": 0.3232,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.26833707094192505,
      "rewards/margins": 1.1447843313217163,
      "rewards/rejected": -1.4131214618682861,
      "step": 840
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.5569917743830786e-07,
      "logits/chosen": -2.5251667499542236,
      "logits/rejected": -2.675060510635376,
      "logps/chosen": -251.2278594970703,
      "logps/rejected": -315.56207275390625,
      "loss": 0.293,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.25193172693252563,
      "rewards/margins": 1.334147334098816,
      "rewards/rejected": -1.5860790014266968,
      "step": 850
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.54524089306698e-07,
      "logits/chosen": -2.5597708225250244,
      "logits/rejected": -2.753092050552368,
      "logps/chosen": -211.96786499023438,
      "logps/rejected": -299.43524169921875,
      "loss": 0.2799,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.19223108887672424,
      "rewards/margins": 1.4486385583877563,
      "rewards/rejected": -1.6408697366714478,
      "step": 860
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.533490011750881e-07,
      "logits/chosen": -2.4884285926818848,
      "logits/rejected": -2.6827969551086426,
      "logps/chosen": -249.5803985595703,
      "logps/rejected": -313.22320556640625,
      "loss": 0.2979,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.24768991768360138,
      "rewards/margins": 1.3257328271865845,
      "rewards/rejected": -1.5734227895736694,
      "step": 870
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.521739130434782e-07,
      "logits/chosen": -2.4708611965179443,
      "logits/rejected": -2.640122652053833,
      "logps/chosen": -266.4299621582031,
      "logps/rejected": -334.88525390625,
      "loss": 0.2825,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.2866387963294983,
      "rewards/margins": 1.3612065315246582,
      "rewards/rejected": -1.6478450298309326,
      "step": 880
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.5099882491186837e-07,
      "logits/chosen": -2.5099267959594727,
      "logits/rejected": -2.6712393760681152,
      "logps/chosen": -234.1786346435547,
      "logps/rejected": -309.02423095703125,
      "loss": 0.299,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3090313971042633,
      "rewards/margins": 1.3444072008132935,
      "rewards/rejected": -1.6534388065338135,
      "step": 890
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.4982373678025847e-07,
      "logits/chosen": -2.5013184547424316,
      "logits/rejected": -2.612173557281494,
      "logps/chosen": -287.0076904296875,
      "logps/rejected": -314.8323059082031,
      "loss": 0.3035,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.217021182179451,
      "rewards/margins": 1.3306859731674194,
      "rewards/rejected": -1.547707200050354,
      "step": 900
    },
    {
      "epoch": 0.57,
      "eval_logits/chosen": -2.6409823894500732,
      "eval_logits/rejected": -2.773390054702759,
      "eval_logps/chosen": -273.8271789550781,
      "eval_logps/rejected": -302.6710205078125,
      "eval_loss": 0.3094734251499176,
      "eval_rewards/accuracies": 0.8307692408561707,
      "eval_rewards/chosen": -0.2914349436759949,
      "eval_rewards/margins": 1.3048936128616333,
      "eval_rewards/rejected": -1.5963284969329834,
      "eval_runtime": 108.2614,
      "eval_samples_per_second": 2.383,
      "eval_steps_per_second": 0.6,
      "step": 900
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.486486486486487e-07,
      "logits/chosen": -2.593217134475708,
      "logits/rejected": -2.708028793334961,
      "logps/chosen": -245.435302734375,
      "logps/rejected": -285.18414306640625,
      "loss": 0.3098,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.2960805892944336,
      "rewards/margins": 1.306485891342163,
      "rewards/rejected": -1.6025664806365967,
      "step": 910
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.474735605170388e-07,
      "logits/chosen": -2.5899627208709717,
      "logits/rejected": -2.7135205268859863,
      "logps/chosen": -236.89859008789062,
      "logps/rejected": -303.027099609375,
      "loss": 0.3255,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.22707653045654297,
      "rewards/margins": 1.2563432455062866,
      "rewards/rejected": -1.4834197759628296,
      "step": 920
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.462984723854289e-07,
      "logits/chosen": -2.4754669666290283,
      "logits/rejected": -2.6056361198425293,
      "logps/chosen": -270.06341552734375,
      "logps/rejected": -309.84405517578125,
      "loss": 0.282,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.28730931878089905,
      "rewards/margins": 1.4082820415496826,
      "rewards/rejected": -1.6955915689468384,
      "step": 930
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.4512338425381903e-07,
      "logits/chosen": -2.6033434867858887,
      "logits/rejected": -2.737351894378662,
      "logps/chosen": -250.7384490966797,
      "logps/rejected": -274.4999084472656,
      "loss": 0.3217,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.3605068325996399,
      "rewards/margins": 1.23686683177948,
      "rewards/rejected": -1.5973737239837646,
      "step": 940
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.4394829612220913e-07,
      "logits/chosen": -2.5599379539489746,
      "logits/rejected": -2.703143358230591,
      "logps/chosen": -221.07034301757812,
      "logps/rejected": -284.4543151855469,
      "loss": 0.3048,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.2962890565395355,
      "rewards/margins": 1.374326467514038,
      "rewards/rejected": -1.670615553855896,
      "step": 950
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.4277320799059924e-07,
      "logits/chosen": -2.4750888347625732,
      "logits/rejected": -2.622952699661255,
      "logps/chosen": -285.9586486816406,
      "logps/rejected": -358.65484619140625,
      "loss": 0.2507,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3137187361717224,
      "rewards/margins": 1.5902440547943115,
      "rewards/rejected": -1.9039628505706787,
      "step": 960
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.4159811985898944e-07,
      "logits/chosen": -2.6037821769714355,
      "logits/rejected": -2.74092435836792,
      "logps/chosen": -297.5964050292969,
      "logps/rejected": -339.80078125,
      "loss": 0.3214,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2947555184364319,
      "rewards/margins": 1.324500322341919,
      "rewards/rejected": -1.619255781173706,
      "step": 970
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.4042303172737954e-07,
      "logits/chosen": -2.5185439586639404,
      "logits/rejected": -2.6582367420196533,
      "logps/chosen": -232.27267456054688,
      "logps/rejected": -296.9894714355469,
      "loss": 0.2963,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.26027384400367737,
      "rewards/margins": 1.3402659893035889,
      "rewards/rejected": -1.6005401611328125,
      "step": 980
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.3924794359576964e-07,
      "logits/chosen": -2.60060453414917,
      "logits/rejected": -2.7379021644592285,
      "logps/chosen": -263.73883056640625,
      "logps/rejected": -332.8947448730469,
      "loss": 0.2771,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2992052733898163,
      "rewards/margins": 1.588271975517273,
      "rewards/rejected": -1.8874772787094116,
      "step": 990
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.380728554641598e-07,
      "logits/chosen": -2.5424516201019287,
      "logits/rejected": -2.685253143310547,
      "logps/chosen": -249.15719604492188,
      "logps/rejected": -302.11480712890625,
      "loss": 0.2565,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.3770313858985901,
      "rewards/margins": 1.5861696004867554,
      "rewards/rejected": -1.9632009267807007,
      "step": 1000
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.639681816101074,
      "eval_logits/rejected": -2.771221399307251,
      "eval_logps/chosen": -274.2304992675781,
      "eval_logps/rejected": -304.87060546875,
      "eval_loss": 0.2856278419494629,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.3317677676677704,
      "eval_rewards/margins": 1.4845184087753296,
      "eval_rewards/rejected": -1.8162860870361328,
      "eval_runtime": 106.5608,
      "eval_samples_per_second": 2.421,
      "eval_steps_per_second": 0.61,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.368977673325499e-07,
      "logits/chosen": -2.5963921546936035,
      "logits/rejected": -2.734703779220581,
      "logps/chosen": -275.4571533203125,
      "logps/rejected": -329.7334899902344,
      "loss": 0.2976,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.34512192010879517,
      "rewards/margins": 1.50485360622406,
      "rewards/rejected": -1.8499755859375,
      "step": 1010
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.3572267920094e-07,
      "logits/chosen": -2.4954187870025635,
      "logits/rejected": -2.628837823867798,
      "logps/chosen": -255.601806640625,
      "logps/rejected": -292.4178161621094,
      "loss": 0.2436,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.32373347878456116,
      "rewards/margins": 1.6133304834365845,
      "rewards/rejected": -1.9370639324188232,
      "step": 1020
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.345475910693302e-07,
      "logits/chosen": -2.5155205726623535,
      "logits/rejected": -2.6221213340759277,
      "logps/chosen": -278.9083557128906,
      "logps/rejected": -325.02215576171875,
      "loss": 0.2743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3400457799434662,
      "rewards/margins": 1.535178303718567,
      "rewards/rejected": -1.8752241134643555,
      "step": 1030
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.333725029377203e-07,
      "logits/chosen": -2.4966979026794434,
      "logits/rejected": -2.610551357269287,
      "logps/chosen": -225.76925659179688,
      "logps/rejected": -291.86065673828125,
      "loss": 0.2723,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.2749883830547333,
      "rewards/margins": 1.5691866874694824,
      "rewards/rejected": -1.8441749811172485,
      "step": 1040
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.3219741480611046e-07,
      "logits/chosen": -2.549757719039917,
      "logits/rejected": -2.663745641708374,
      "logps/chosen": -261.6253967285156,
      "logps/rejected": -307.04425048828125,
      "loss": 0.2698,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.34586817026138306,
      "rewards/margins": 1.612839937210083,
      "rewards/rejected": -1.9587081670761108,
      "step": 1050
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.3102232667450057e-07,
      "logits/chosen": -2.545283794403076,
      "logits/rejected": -2.6645588874816895,
      "logps/chosen": -271.39410400390625,
      "logps/rejected": -323.82196044921875,
      "loss": 0.2398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40222612023353577,
      "rewards/margins": 1.7886310815811157,
      "rewards/rejected": -2.190857172012329,
      "step": 1060
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.2984723854289067e-07,
      "logits/chosen": -2.498837947845459,
      "logits/rejected": -2.631861686706543,
      "logps/chosen": -276.1019592285156,
      "logps/rejected": -309.6996765136719,
      "loss": 0.2223,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.3812776207923889,
      "rewards/margins": 1.7885946035385132,
      "rewards/rejected": -2.1698720455169678,
      "step": 1070
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.286721504112809e-07,
      "logits/chosen": -2.4292163848876953,
      "logits/rejected": -2.613614797592163,
      "logps/chosen": -219.46493530273438,
      "logps/rejected": -303.3463134765625,
      "loss": 0.228,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.37212830781936646,
      "rewards/margins": 1.801314115524292,
      "rewards/rejected": -2.1734423637390137,
      "step": 1080
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.27497062279671e-07,
      "logits/chosen": -2.550693988800049,
      "logits/rejected": -2.6413536071777344,
      "logps/chosen": -291.6618347167969,
      "logps/rejected": -312.3702392578125,
      "loss": 0.2662,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.43923258781433105,
      "rewards/margins": 1.5901190042495728,
      "rewards/rejected": -2.0293517112731934,
      "step": 1090
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.263219741480611e-07,
      "logits/chosen": -2.501007080078125,
      "logits/rejected": -2.6585614681243896,
      "logps/chosen": -256.6453552246094,
      "logps/rejected": -318.043701171875,
      "loss": 0.2409,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.35768821835517883,
      "rewards/margins": 1.7618768215179443,
      "rewards/rejected": -2.11956524848938,
      "step": 1100
    },
    {
      "epoch": 0.7,
      "eval_logits/chosen": -2.637993097305298,
      "eval_logits/rejected": -2.769087314605713,
      "eval_logps/chosen": -274.6673278808594,
      "eval_logps/rejected": -306.9071350097656,
      "eval_loss": 0.2676018178462982,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.37544897198677063,
      "eval_rewards/margins": 1.6444897651672363,
      "eval_rewards/rejected": -2.0199387073516846,
      "eval_runtime": 104.9523,
      "eval_samples_per_second": 2.458,
      "eval_steps_per_second": 0.619,
      "step": 1100
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.2514688601645123e-07,
      "logits/chosen": -2.5811259746551514,
      "logits/rejected": -2.7109222412109375,
      "logps/chosen": -295.91595458984375,
      "logps/rejected": -317.20098876953125,
      "loss": 0.2664,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3150026202201843,
      "rewards/margins": 1.6915771961212158,
      "rewards/rejected": -2.006579875946045,
      "step": 1110
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.2397179788484133e-07,
      "logits/chosen": -2.534104108810425,
      "logits/rejected": -2.6629557609558105,
      "logps/chosen": -226.57699584960938,
      "logps/rejected": -284.3053283691406,
      "loss": 0.268,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.40733012557029724,
      "rewards/margins": 1.6161415576934814,
      "rewards/rejected": -2.0234715938568115,
      "step": 1120
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.2279670975323143e-07,
      "logits/chosen": -2.5734691619873047,
      "logits/rejected": -2.6925172805786133,
      "logps/chosen": -258.71783447265625,
      "logps/rejected": -312.15386962890625,
      "loss": 0.2856,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3448827266693115,
      "rewards/margins": 1.647994041442871,
      "rewards/rejected": -1.992876648902893,
      "step": 1130
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.2162162162162164e-07,
      "logits/chosen": -2.563662052154541,
      "logits/rejected": -2.682953119277954,
      "logps/chosen": -245.13638305664062,
      "logps/rejected": -292.2523193359375,
      "loss": 0.2673,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.28280961513519287,
      "rewards/margins": 1.7050749063491821,
      "rewards/rejected": -1.987884521484375,
      "step": 1140
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.2044653349001174e-07,
      "logits/chosen": -2.633129835128784,
      "logits/rejected": -2.747661590576172,
      "logps/chosen": -239.8200225830078,
      "logps/rejected": -278.12493896484375,
      "loss": 0.28,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.312093049287796,
      "rewards/margins": 1.588714838027954,
      "rewards/rejected": -1.9008080959320068,
      "step": 1150
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.1927144535840184e-07,
      "logits/chosen": -2.5131030082702637,
      "logits/rejected": -2.621605396270752,
      "logps/chosen": -245.6011199951172,
      "logps/rejected": -289.03070068359375,
      "loss": 0.2603,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.44687962532043457,
      "rewards/margins": 1.6997129917144775,
      "rewards/rejected": -2.146592617034912,
      "step": 1160
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.18096357226792e-07,
      "logits/chosen": -2.513087749481201,
      "logits/rejected": -2.6465277671813965,
      "logps/chosen": -223.76583862304688,
      "logps/rejected": -283.69708251953125,
      "loss": 0.2292,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.39333704113960266,
      "rewards/margins": 1.7673060894012451,
      "rewards/rejected": -2.1606431007385254,
      "step": 1170
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.169212690951821e-07,
      "logits/chosen": -2.6067252159118652,
      "logits/rejected": -2.7137060165405273,
      "logps/chosen": -267.8930969238281,
      "logps/rejected": -314.7989196777344,
      "loss": 0.251,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.46876877546310425,
      "rewards/margins": 1.7331355810165405,
      "rewards/rejected": -2.201904535293579,
      "step": 1180
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.157461809635722e-07,
      "logits/chosen": -2.5227315425872803,
      "logits/rejected": -2.6699652671813965,
      "logps/chosen": -254.6536102294922,
      "logps/rejected": -315.83624267578125,
      "loss": 0.1866,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.33257487416267395,
      "rewards/margins": 2.179680347442627,
      "rewards/rejected": -2.5122549533843994,
      "step": 1190
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.145710928319624e-07,
      "logits/chosen": -2.563264846801758,
      "logits/rejected": -2.6574532985687256,
      "logps/chosen": -318.592041015625,
      "logps/rejected": -339.0600891113281,
      "loss": 0.2341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4399125576019287,
      "rewards/margins": 1.7574468851089478,
      "rewards/rejected": -2.197359323501587,
      "step": 1200
    },
    {
      "epoch": 0.76,
      "eval_logits/chosen": -2.6370952129364014,
      "eval_logits/rejected": -2.76749849319458,
      "eval_logps/chosen": -275.1463317871094,
      "eval_logps/rejected": -308.983154296875,
      "eval_loss": 0.2514963150024414,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.42334869503974915,
      "eval_rewards/margins": 1.804190754890442,
      "eval_rewards/rejected": -2.227539300918579,
      "eval_runtime": 107.9501,
      "eval_samples_per_second": 2.39,
      "eval_steps_per_second": 0.602,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.133960047003525e-07,
      "logits/chosen": -2.606009006500244,
      "logits/rejected": -2.7354376316070557,
      "logps/chosen": -291.32940673828125,
      "logps/rejected": -351.57080078125,
      "loss": 0.2269,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.3423115313053131,
      "rewards/margins": 2.1125004291534424,
      "rewards/rejected": -2.4548118114471436,
      "step": 1210
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.1222091656874266e-07,
      "logits/chosen": -2.545210361480713,
      "logits/rejected": -2.64381742477417,
      "logps/chosen": -286.2717590332031,
      "logps/rejected": -316.93145751953125,
      "loss": 0.2731,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.33074694871902466,
      "rewards/margins": 1.7963011264801025,
      "rewards/rejected": -2.1270480155944824,
      "step": 1220
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.1104582843713276e-07,
      "logits/chosen": -2.565667152404785,
      "logits/rejected": -2.6980319023132324,
      "logps/chosen": -249.2791748046875,
      "logps/rejected": -304.4717102050781,
      "loss": 0.2257,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.40403205156326294,
      "rewards/margins": 1.9614498615264893,
      "rewards/rejected": -2.3654818534851074,
      "step": 1230
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.0987074030552287e-07,
      "logits/chosen": -2.457113742828369,
      "logits/rejected": -2.6211788654327393,
      "logps/chosen": -257.15380859375,
      "logps/rejected": -309.3705749511719,
      "loss": 0.2059,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.38037046790122986,
      "rewards/margins": 2.0390753746032715,
      "rewards/rejected": -2.419445753097534,
      "step": 1240
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.0869565217391307e-07,
      "logits/chosen": -2.434441089630127,
      "logits/rejected": -2.5575058460235596,
      "logps/chosen": -274.3152160644531,
      "logps/rejected": -334.8428955078125,
      "loss": 0.1741,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4384829103946686,
      "rewards/margins": 2.2238662242889404,
      "rewards/rejected": -2.662349224090576,
      "step": 1250
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.075205640423032e-07,
      "logits/chosen": -2.4969711303710938,
      "logits/rejected": -2.654552459716797,
      "logps/chosen": -230.37319946289062,
      "logps/rejected": -317.71624755859375,
      "loss": 0.1736,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5470365881919861,
      "rewards/margins": 2.293950080871582,
      "rewards/rejected": -2.840986728668213,
      "step": 1260
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.063454759106933e-07,
      "logits/chosen": -2.4980969429016113,
      "logits/rejected": -2.6356616020202637,
      "logps/chosen": -262.65618896484375,
      "logps/rejected": -312.2105407714844,
      "loss": 0.2329,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.47642117738723755,
      "rewards/margins": 2.0892834663391113,
      "rewards/rejected": -2.565704584121704,
      "step": 1270
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.0517038777908343e-07,
      "logits/chosen": -2.565469980239868,
      "logits/rejected": -2.6740055084228516,
      "logps/chosen": -271.97967529296875,
      "logps/rejected": -323.845458984375,
      "loss": 0.2199,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4494257867336273,
      "rewards/margins": 1.991673469543457,
      "rewards/rejected": -2.4410996437072754,
      "step": 1280
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.0399529964747353e-07,
      "logits/chosen": -2.5231869220733643,
      "logits/rejected": -2.6080517768859863,
      "logps/chosen": -266.5882873535156,
      "logps/rejected": -314.27642822265625,
      "loss": 0.218,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4522724747657776,
      "rewards/margins": 2.1093709468841553,
      "rewards/rejected": -2.561643362045288,
      "step": 1290
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.0282021151586363e-07,
      "logits/chosen": -2.558234214782715,
      "logits/rejected": -2.6852424144744873,
      "logps/chosen": -221.29708862304688,
      "logps/rejected": -291.7198486328125,
      "loss": 0.2584,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.45204487442970276,
      "rewards/margins": 1.8207969665527344,
      "rewards/rejected": -2.272841691970825,
      "step": 1300
    },
    {
      "epoch": 0.82,
      "eval_logits/chosen": -2.6354877948760986,
      "eval_logits/rejected": -2.7653422355651855,
      "eval_logps/chosen": -275.7122802734375,
      "eval_logps/rejected": -311.00823974609375,
      "eval_loss": 0.23934800922870636,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.4799441397190094,
      "eval_rewards/margins": 1.9501070976257324,
      "eval_rewards/rejected": -2.430050849914551,
      "eval_runtime": 102.8577,
      "eval_samples_per_second": 2.508,
      "eval_steps_per_second": 0.632,
      "step": 1300
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0164512338425384e-07,
      "logits/chosen": -2.5574240684509277,
      "logits/rejected": -2.7002782821655273,
      "logps/chosen": -257.2291259765625,
      "logps/rejected": -323.4848937988281,
      "loss": 0.2371,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.45065683126449585,
      "rewards/margins": 2.009228229522705,
      "rewards/rejected": -2.4598851203918457,
      "step": 1310
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.0047003525264394e-07,
      "logits/chosen": -2.4942541122436523,
      "logits/rejected": -2.591789722442627,
      "logps/chosen": -280.63677978515625,
      "logps/rejected": -320.3000793457031,
      "loss": 0.2008,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6209262609481812,
      "rewards/margins": 2.050626277923584,
      "rewards/rejected": -2.6715526580810547,
      "step": 1320
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9929494712103404e-07,
      "logits/chosen": -2.5325863361358643,
      "logits/rejected": -2.6914854049682617,
      "logps/chosen": -224.44931030273438,
      "logps/rejected": -305.97314453125,
      "loss": 0.2205,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4621918797492981,
      "rewards/margins": 2.0263607501983643,
      "rewards/rejected": -2.4885528087615967,
      "step": 1330
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.981198589894242e-07,
      "logits/chosen": -2.547691822052002,
      "logits/rejected": -2.6408214569091797,
      "logps/chosen": -269.5528564453125,
      "logps/rejected": -311.5650329589844,
      "loss": 0.2111,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5892491936683655,
      "rewards/margins": 2.106876850128174,
      "rewards/rejected": -2.6961262226104736,
      "step": 1340
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.969447708578143e-07,
      "logits/chosen": -2.6148841381073,
      "logits/rejected": -2.6921582221984863,
      "logps/chosen": -237.748046875,
      "logps/rejected": -266.81341552734375,
      "loss": 0.2291,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48353129625320435,
      "rewards/margins": 1.9532550573349,
      "rewards/rejected": -2.43678617477417,
      "step": 1350
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.957696827262044e-07,
      "logits/chosen": -2.4597270488739014,
      "logits/rejected": -2.6138980388641357,
      "logps/chosen": -244.71804809570312,
      "logps/rejected": -311.88763427734375,
      "loss": 0.2024,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.544535219669342,
      "rewards/margins": 2.2707481384277344,
      "rewards/rejected": -2.8152832984924316,
      "step": 1360
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.945945945945946e-07,
      "logits/chosen": -2.6130173206329346,
      "logits/rejected": -2.7234058380126953,
      "logps/chosen": -335.40545654296875,
      "logps/rejected": -374.2771301269531,
      "loss": 0.2171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6030958890914917,
      "rewards/margins": 2.044811248779297,
      "rewards/rejected": -2.647907257080078,
      "step": 1370
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.934195064629847e-07,
      "logits/chosen": -2.5572052001953125,
      "logits/rejected": -2.661154270172119,
      "logps/chosen": -264.9273986816406,
      "logps/rejected": -305.95208740234375,
      "loss": 0.2234,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6006981134414673,
      "rewards/margins": 2.090214967727661,
      "rewards/rejected": -2.690912961959839,
      "step": 1380
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9224441833137486e-07,
      "logits/chosen": -2.564368486404419,
      "logits/rejected": -2.666646957397461,
      "logps/chosen": -271.6180725097656,
      "logps/rejected": -307.5677185058594,
      "loss": 0.184,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5409923791885376,
      "rewards/margins": 2.195977210998535,
      "rewards/rejected": -2.736969470977783,
      "step": 1390
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.9106933019976496e-07,
      "logits/chosen": -2.500899314880371,
      "logits/rejected": -2.6401665210723877,
      "logps/chosen": -280.5640563964844,
      "logps/rejected": -357.0772705078125,
      "loss": 0.2171,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5830007791519165,
      "rewards/margins": 2.1108529567718506,
      "rewards/rejected": -2.6938538551330566,
      "step": 1400
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": -2.634171962738037,
      "eval_logits/rejected": -2.7634615898132324,
      "eval_logps/chosen": -276.1873474121094,
      "eval_logps/rejected": -312.79443359375,
      "eval_loss": 0.22938059270381927,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.5274493098258972,
      "eval_rewards/margins": 2.0812206268310547,
      "eval_rewards/rejected": -2.6086697578430176,
      "eval_runtime": 102.7756,
      "eval_samples_per_second": 2.51,
      "eval_steps_per_second": 0.632,
      "step": 1400
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.8989424206815507e-07,
      "logits/chosen": -2.42073655128479,
      "logits/rejected": -2.58854079246521,
      "logps/chosen": -243.45535278320312,
      "logps/rejected": -335.9302978515625,
      "loss": 0.1701,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4307083487510681,
      "rewards/margins": 2.4085564613342285,
      "rewards/rejected": -2.8392646312713623,
      "step": 1410
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.887191539365452e-07,
      "logits/chosen": -2.448655128479004,
      "logits/rejected": -2.5622687339782715,
      "logps/chosen": -280.87054443359375,
      "logps/rejected": -317.5985412597656,
      "loss": 0.2207,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.5180009603500366,
      "rewards/margins": 2.160266399383545,
      "rewards/rejected": -2.678267478942871,
      "step": 1420
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.8754406580493537e-07,
      "logits/chosen": -2.477066993713379,
      "logits/rejected": -2.6448426246643066,
      "logps/chosen": -192.8367156982422,
      "logps/rejected": -277.80841064453125,
      "loss": 0.185,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4950019419193268,
      "rewards/margins": 2.292844772338867,
      "rewards/rejected": -2.787846803665161,
      "step": 1430
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.863689776733255e-07,
      "logits/chosen": -2.4869980812072754,
      "logits/rejected": -2.6351399421691895,
      "logps/chosen": -234.60989379882812,
      "logps/rejected": -322.6162109375,
      "loss": 0.1892,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4373553693294525,
      "rewards/margins": 2.3803677558898926,
      "rewards/rejected": -2.817723512649536,
      "step": 1440
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.8519388954171563e-07,
      "logits/chosen": -2.58585262298584,
      "logits/rejected": -2.6931519508361816,
      "logps/chosen": -233.1945343017578,
      "logps/rejected": -287.9420471191406,
      "loss": 0.2056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5764862895011902,
      "rewards/margins": 2.1881613731384277,
      "rewards/rejected": -2.764647960662842,
      "step": 1450
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.8401880141010573e-07,
      "logits/chosen": -2.492675542831421,
      "logits/rejected": -2.61580228805542,
      "logps/chosen": -207.68588256835938,
      "logps/rejected": -284.49029541015625,
      "loss": 0.2018,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5032284259796143,
      "rewards/margins": 2.2765746116638184,
      "rewards/rejected": -2.7798032760620117,
      "step": 1460
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.8284371327849583e-07,
      "logits/chosen": -2.6050140857696533,
      "logits/rejected": -2.7160239219665527,
      "logps/chosen": -280.0237731933594,
      "logps/rejected": -327.25421142578125,
      "loss": 0.2108,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5926626324653625,
      "rewards/margins": 2.2293572425842285,
      "rewards/rejected": -2.8220200538635254,
      "step": 1470
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.8166862514688604e-07,
      "logits/chosen": -2.446561813354492,
      "logits/rejected": -2.59938383102417,
      "logps/chosen": -238.4915313720703,
      "logps/rejected": -318.1725769042969,
      "loss": 0.2008,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.5125645995140076,
      "rewards/margins": 2.4164915084838867,
      "rewards/rejected": -2.929055690765381,
      "step": 1480
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.8049353701527614e-07,
      "logits/chosen": -2.623857021331787,
      "logits/rejected": -2.736724376678467,
      "logps/chosen": -238.219970703125,
      "logps/rejected": -304.45159912109375,
      "loss": 0.2362,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.492041677236557,
      "rewards/margins": 2.1697030067443848,
      "rewards/rejected": -2.6617445945739746,
      "step": 1490
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.7931844888366624e-07,
      "logits/chosen": -2.5280914306640625,
      "logits/rejected": -2.7397713661193848,
      "logps/chosen": -249.3155517578125,
      "logps/rejected": -350.5032653808594,
      "loss": 0.1638,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5685161352157593,
      "rewards/margins": 2.718121290206909,
      "rewards/rejected": -3.286637544631958,
      "step": 1500
    },
    {
      "epoch": 0.95,
      "eval_logits/chosen": -2.6335904598236084,
      "eval_logits/rejected": -2.7623374462127686,
      "eval_logps/chosen": -276.6611328125,
      "eval_logps/rejected": -314.60205078125,
      "eval_loss": 0.2205849289894104,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.5748318433761597,
      "eval_rewards/margins": 2.214599132537842,
      "eval_rewards/rejected": -2.789431095123291,
      "eval_runtime": 109.2712,
      "eval_samples_per_second": 2.361,
      "eval_steps_per_second": 0.595,
      "step": 1500
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.781433607520564e-07,
      "logits/chosen": -2.5514187812805176,
      "logits/rejected": -2.704192876815796,
      "logps/chosen": -253.22366333007812,
      "logps/rejected": -315.31890869140625,
      "loss": 0.1895,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.41746625304222107,
      "rewards/margins": 2.4687085151672363,
      "rewards/rejected": -2.8861749172210693,
      "step": 1510
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.769682726204465e-07,
      "logits/chosen": -2.680431842803955,
      "logits/rejected": -2.7628579139709473,
      "logps/chosen": -261.89776611328125,
      "logps/rejected": -309.6994323730469,
      "loss": 0.2395,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7225305438041687,
      "rewards/margins": 2.0975546836853027,
      "rewards/rejected": -2.820085048675537,
      "step": 1520
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.757931844888366e-07,
      "logits/chosen": -2.571585178375244,
      "logits/rejected": -2.685753345489502,
      "logps/chosen": -294.14764404296875,
      "logps/rejected": -348.1631774902344,
      "loss": 0.2355,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6126054525375366,
      "rewards/margins": 2.301832914352417,
      "rewards/rejected": -2.9144387245178223,
      "step": 1530
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.746180963572268e-07,
      "logits/chosen": -2.510603189468384,
      "logits/rejected": -2.6411643028259277,
      "logps/chosen": -217.03372192382812,
      "logps/rejected": -288.92401123046875,
      "loss": 0.2272,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4745887815952301,
      "rewards/margins": 2.2049689292907715,
      "rewards/rejected": -2.6795575618743896,
      "step": 1540
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.734430082256169e-07,
      "logits/chosen": -2.5020484924316406,
      "logits/rejected": -2.634915828704834,
      "logps/chosen": -212.1004180908203,
      "logps/rejected": -305.2049865722656,
      "loss": 0.1878,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5657206773757935,
      "rewards/margins": 2.333150863647461,
      "rewards/rejected": -2.898871898651123,
      "step": 1550
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7226792009400706e-07,
      "logits/chosen": -2.5634593963623047,
      "logits/rejected": -2.717979907989502,
      "logps/chosen": -258.0527648925781,
      "logps/rejected": -338.5130920410156,
      "loss": 0.1853,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6402656435966492,
      "rewards/margins": 2.4888405799865723,
      "rewards/rejected": -3.129106044769287,
      "step": 1560
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.7109283196239716e-07,
      "logits/chosen": -2.4743266105651855,
      "logits/rejected": -2.642153024673462,
      "logps/chosen": -244.82760620117188,
      "logps/rejected": -343.3356628417969,
      "loss": 0.1701,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5895872712135315,
      "rewards/margins": 2.6921560764312744,
      "rewards/rejected": -3.281743288040161,
      "step": 1570
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.6991774383078726e-07,
      "logits/chosen": -2.540102481842041,
      "logits/rejected": -2.717726945877075,
      "logps/chosen": -236.639892578125,
      "logps/rejected": -325.9012145996094,
      "loss": 0.1583,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6464608311653137,
      "rewards/margins": 2.8656179904937744,
      "rewards/rejected": -3.5120785236358643,
      "step": 1580
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.687426556991774e-07,
      "logits/chosen": -2.502453327178955,
      "logits/rejected": -2.630316972732544,
      "logps/chosen": -277.71746826171875,
      "logps/rejected": -339.40020751953125,
      "loss": 0.2122,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5439355969429016,
      "rewards/margins": 2.3802871704101562,
      "rewards/rejected": -2.924222946166992,
      "step": 1590
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.6756756756756757e-07,
      "logits/chosen": -2.6116623878479004,
      "logits/rejected": -2.743366241455078,
      "logps/chosen": -265.41827392578125,
      "logps/rejected": -327.4100646972656,
      "loss": 0.2334,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5309698581695557,
      "rewards/margins": 2.2668042182922363,
      "rewards/rejected": -2.797774076461792,
      "step": 1600
    },
    {
      "epoch": 1.02,
      "eval_logits/chosen": -2.63187575340271,
      "eval_logits/rejected": -2.7603108882904053,
      "eval_logps/chosen": -277.0210266113281,
      "eval_logps/rejected": -316.0559387207031,
      "eval_loss": 0.21472880244255066,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.6108187437057495,
      "eval_rewards/margins": 2.324002265930176,
      "eval_rewards/rejected": -2.934821367263794,
      "eval_runtime": 105.9847,
      "eval_samples_per_second": 2.434,
      "eval_steps_per_second": 0.613,
      "step": 1600
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.663924794359577e-07,
      "logits/chosen": -2.512497663497925,
      "logits/rejected": -2.6775221824645996,
      "logps/chosen": -213.3688201904297,
      "logps/rejected": -293.5090026855469,
      "loss": 0.2247,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.4678749144077301,
      "rewards/margins": 2.3030238151550293,
      "rewards/rejected": -2.7708992958068848,
      "step": 1610
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.6521739130434783e-07,
      "logits/chosen": -2.503650188446045,
      "logits/rejected": -2.6239073276519775,
      "logps/chosen": -262.0201721191406,
      "logps/rejected": -334.70257568359375,
      "loss": 0.1593,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7141811847686768,
      "rewards/margins": 2.8270955085754395,
      "rewards/rejected": -3.541276454925537,
      "step": 1620
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.6404230317273793e-07,
      "logits/chosen": -2.4200310707092285,
      "logits/rejected": -2.598867177963257,
      "logps/chosen": -208.276123046875,
      "logps/rejected": -307.66473388671875,
      "loss": 0.16,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5537343621253967,
      "rewards/margins": 2.8423819541931152,
      "rewards/rejected": -3.396116256713867,
      "step": 1630
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.6286721504112803e-07,
      "logits/chosen": -2.5602195262908936,
      "logits/rejected": -2.639371395111084,
      "logps/chosen": -294.02789306640625,
      "logps/rejected": -322.9757995605469,
      "loss": 0.2089,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6427415609359741,
      "rewards/margins": 2.328885555267334,
      "rewards/rejected": -2.9716269969940186,
      "step": 1640
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.6169212690951824e-07,
      "logits/chosen": -2.5637240409851074,
      "logits/rejected": -2.6973021030426025,
      "logps/chosen": -283.67730712890625,
      "logps/rejected": -329.50030517578125,
      "loss": 0.1948,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7129117250442505,
      "rewards/margins": 2.4820034503936768,
      "rewards/rejected": -3.1949150562286377,
      "step": 1650
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.6051703877790834e-07,
      "logits/chosen": -2.5456652641296387,
      "logits/rejected": -2.6587166786193848,
      "logps/chosen": -302.8798522949219,
      "logps/rejected": -332.8749694824219,
      "loss": 0.1734,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5599570870399475,
      "rewards/margins": 2.582099437713623,
      "rewards/rejected": -3.1420562267303467,
      "step": 1660
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.5934195064629844e-07,
      "logits/chosen": -2.500321388244629,
      "logits/rejected": -2.6799192428588867,
      "logps/chosen": -298.4835205078125,
      "logps/rejected": -366.3076477050781,
      "loss": 0.1621,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.5673084855079651,
      "rewards/margins": 2.773592710494995,
      "rewards/rejected": -3.3409011363983154,
      "step": 1670
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.581668625146886e-07,
      "logits/chosen": -2.5583109855651855,
      "logits/rejected": -2.6897902488708496,
      "logps/chosen": -244.66635131835938,
      "logps/rejected": -327.6900939941406,
      "loss": 0.2125,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.5903083086013794,
      "rewards/margins": 2.493638753890991,
      "rewards/rejected": -3.08394718170166,
      "step": 1680
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.569917743830787e-07,
      "logits/chosen": -2.580543041229248,
      "logits/rejected": -2.6891369819641113,
      "logps/chosen": -273.4271545410156,
      "logps/rejected": -306.363037109375,
      "loss": 0.1796,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7122648358345032,
      "rewards/margins": 2.406428813934326,
      "rewards/rejected": -3.1186938285827637,
      "step": 1690
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.558166862514688e-07,
      "logits/chosen": -2.512493133544922,
      "logits/rejected": -2.594755172729492,
      "logps/chosen": -296.9983215332031,
      "logps/rejected": -317.21142578125,
      "loss": 0.2178,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6500630378723145,
      "rewards/margins": 2.2964870929718018,
      "rewards/rejected": -2.946549892425537,
      "step": 1700
    },
    {
      "epoch": 1.08,
      "eval_logits/chosen": -2.6313717365264893,
      "eval_logits/rejected": -2.759672164916992,
      "eval_logps/chosen": -277.43548583984375,
      "eval_logps/rejected": -317.4504699707031,
      "eval_loss": 0.208636075258255,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.6522640585899353,
      "eval_rewards/margins": 2.422011613845825,
      "eval_rewards/rejected": -3.074275493621826,
      "eval_runtime": 102.8491,
      "eval_samples_per_second": 2.509,
      "eval_steps_per_second": 0.632,
      "step": 1700
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.54641598119859e-07,
      "logits/chosen": -2.580533027648926,
      "logits/rejected": -2.696044921875,
      "logps/chosen": -252.8826141357422,
      "logps/rejected": -318.81573486328125,
      "loss": 0.2146,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7479490041732788,
      "rewards/margins": 2.3817811012268066,
      "rewards/rejected": -3.129729986190796,
      "step": 1710
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.534665099882491e-07,
      "logits/chosen": -2.550356388092041,
      "logits/rejected": -2.6657135486602783,
      "logps/chosen": -268.2378845214844,
      "logps/rejected": -299.86749267578125,
      "loss": 0.2021,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6683725118637085,
      "rewards/margins": 2.401125431060791,
      "rewards/rejected": -3.069498062133789,
      "step": 1720
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.5229142185663926e-07,
      "logits/chosen": -2.514503240585327,
      "logits/rejected": -2.6601500511169434,
      "logps/chosen": -231.42544555664062,
      "logps/rejected": -318.6716003417969,
      "loss": 0.1754,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6692171096801758,
      "rewards/margins": 2.7238271236419678,
      "rewards/rejected": -3.3930447101593018,
      "step": 1730
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.5111633372502936e-07,
      "logits/chosen": -2.620729446411133,
      "logits/rejected": -2.7308976650238037,
      "logps/chosen": -360.8436279296875,
      "logps/rejected": -376.83331298828125,
      "loss": 0.2144,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5347746014595032,
      "rewards/margins": 2.4500014781951904,
      "rewards/rejected": -2.984776020050049,
      "step": 1740
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.4994124559341946e-07,
      "logits/chosen": -2.463991165161133,
      "logits/rejected": -2.563396453857422,
      "logps/chosen": -299.7460021972656,
      "logps/rejected": -331.1080627441406,
      "loss": 0.1657,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6542232036590576,
      "rewards/margins": 2.716874361038208,
      "rewards/rejected": -3.3710970878601074,
      "step": 1750
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.487661574618096e-07,
      "logits/chosen": -2.4982893466949463,
      "logits/rejected": -2.649310350418091,
      "logps/chosen": -278.23687744140625,
      "logps/rejected": -364.14190673828125,
      "loss": 0.1669,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6813815236091614,
      "rewards/margins": 2.8649044036865234,
      "rewards/rejected": -3.546285629272461,
      "step": 1760
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.4759106933019977e-07,
      "logits/chosen": -2.499354600906372,
      "logits/rejected": -2.6530840396881104,
      "logps/chosen": -232.09414672851562,
      "logps/rejected": -320.71923828125,
      "loss": 0.2001,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5775212645530701,
      "rewards/margins": 2.741844415664673,
      "rewards/rejected": -3.3193657398223877,
      "step": 1770
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.4641598119858987e-07,
      "logits/chosen": -2.5437896251678467,
      "logits/rejected": -2.6989777088165283,
      "logps/chosen": -209.7606201171875,
      "logps/rejected": -300.90264892578125,
      "loss": 0.1797,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.758380115032196,
      "rewards/margins": 2.6990487575531006,
      "rewards/rejected": -3.4574286937713623,
      "step": 1780
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.4524089306698003e-07,
      "logits/chosen": -2.533337354660034,
      "logits/rejected": -2.6577115058898926,
      "logps/chosen": -280.2760925292969,
      "logps/rejected": -318.46917724609375,
      "loss": 0.193,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6522279977798462,
      "rewards/margins": 2.625479221343994,
      "rewards/rejected": -3.27770733833313,
      "step": 1790
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.4406580493537013e-07,
      "logits/chosen": -2.4202675819396973,
      "logits/rejected": -2.525690793991089,
      "logps/chosen": -247.52218627929688,
      "logps/rejected": -315.266357421875,
      "loss": 0.1704,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6822296977043152,
      "rewards/margins": 2.740001916885376,
      "rewards/rejected": -3.422231674194336,
      "step": 1800
    },
    {
      "epoch": 1.14,
      "eval_logits/chosen": -2.630898952484131,
      "eval_logits/rejected": -2.758958101272583,
      "eval_logps/chosen": -277.7316589355469,
      "eval_logps/rejected": -318.6625671386719,
      "eval_loss": 0.20372293889522552,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.6818835139274597,
      "eval_rewards/margins": 2.5135958194732666,
      "eval_rewards/rejected": -3.195478916168213,
      "eval_runtime": 103.1558,
      "eval_samples_per_second": 2.501,
      "eval_steps_per_second": 0.63,
      "step": 1800
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.4289071680376023e-07,
      "logits/chosen": -2.5175633430480957,
      "logits/rejected": -2.6226401329040527,
      "logps/chosen": -237.10971069335938,
      "logps/rejected": -305.60015869140625,
      "loss": 0.1954,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.5577880144119263,
      "rewards/margins": 2.6116838455200195,
      "rewards/rejected": -3.1694719791412354,
      "step": 1810
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.417156286721504e-07,
      "logits/chosen": -2.5715250968933105,
      "logits/rejected": -2.6752352714538574,
      "logps/chosen": -280.7261657714844,
      "logps/rejected": -327.8868408203125,
      "loss": 0.1885,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6718424558639526,
      "rewards/margins": 2.570451259613037,
      "rewards/rejected": -3.2422938346862793,
      "step": 1820
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.4054054054054054e-07,
      "logits/chosen": -2.4684033393859863,
      "logits/rejected": -2.5957369804382324,
      "logps/chosen": -302.5030212402344,
      "logps/rejected": -360.47607421875,
      "loss": 0.1454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6838303804397583,
      "rewards/margins": 3.0164988040924072,
      "rewards/rejected": -3.700328826904297,
      "step": 1830
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.3936545240893064e-07,
      "logits/chosen": -2.5618185997009277,
      "logits/rejected": -2.701917886734009,
      "logps/chosen": -259.9933776855469,
      "logps/rejected": -320.7430725097656,
      "loss": 0.1824,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.510877788066864,
      "rewards/margins": 2.5309667587280273,
      "rewards/rejected": -3.041844606399536,
      "step": 1840
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.381903642773208e-07,
      "logits/chosen": -2.6064860820770264,
      "logits/rejected": -2.7268707752227783,
      "logps/chosen": -244.10385131835938,
      "logps/rejected": -304.8720703125,
      "loss": 0.2175,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.6699651479721069,
      "rewards/margins": 2.437056541442871,
      "rewards/rejected": -3.1070218086242676,
      "step": 1850
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.370152761457109e-07,
      "logits/chosen": -2.639857530593872,
      "logits/rejected": -2.699920177459717,
      "logps/chosen": -264.599853515625,
      "logps/rejected": -294.0104064941406,
      "loss": 0.2001,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.6109203100204468,
      "rewards/margins": 2.5747509002685547,
      "rewards/rejected": -3.18567156791687,
      "step": 1860
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.35840188014101e-07,
      "logits/chosen": -2.5574636459350586,
      "logits/rejected": -2.6812334060668945,
      "logps/chosen": -325.22918701171875,
      "logps/rejected": -392.3985595703125,
      "loss": 0.1498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9695943593978882,
      "rewards/margins": 2.965458631515503,
      "rewards/rejected": -3.9350533485412598,
      "step": 1870
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.346650998824912e-07,
      "logits/chosen": -2.4894185066223145,
      "logits/rejected": -2.6536877155303955,
      "logps/chosen": -239.0066680908203,
      "logps/rejected": -318.2803039550781,
      "loss": 0.1435,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7069042921066284,
      "rewards/margins": 2.9710018634796143,
      "rewards/rejected": -3.6779067516326904,
      "step": 1880
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.334900117508813e-07,
      "logits/chosen": -2.5469226837158203,
      "logits/rejected": -2.666538715362549,
      "logps/chosen": -223.6966552734375,
      "logps/rejected": -290.5748291015625,
      "loss": 0.1914,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.71305912733078,
      "rewards/margins": 2.4972429275512695,
      "rewards/rejected": -3.210301637649536,
      "step": 1890
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.3231492361927146e-07,
      "logits/chosen": -2.598342180252075,
      "logits/rejected": -2.739816188812256,
      "logps/chosen": -263.1295471191406,
      "logps/rejected": -318.683349609375,
      "loss": 0.1683,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6904613375663757,
      "rewards/margins": 2.79184627532959,
      "rewards/rejected": -3.4823074340820312,
      "step": 1900
    },
    {
      "epoch": 1.21,
      "eval_logits/chosen": -2.6312639713287354,
      "eval_logits/rejected": -2.758720874786377,
      "eval_logps/chosen": -278.0645751953125,
      "eval_logps/rejected": -319.883544921875,
      "eval_loss": 0.19963347911834717,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.7151739001274109,
      "eval_rewards/margins": 2.602405309677124,
      "eval_rewards/rejected": -3.3175787925720215,
      "eval_runtime": 109.1763,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.595,
      "step": 1900
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.3113983548766156e-07,
      "logits/chosen": -2.4476232528686523,
      "logits/rejected": -2.5809733867645264,
      "logps/chosen": -251.45849609375,
      "logps/rejected": -319.7317810058594,
      "loss": 0.1206,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6280637383460999,
      "rewards/margins": 2.971045970916748,
      "rewards/rejected": -3.599109649658203,
      "step": 1910
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.2996474735605166e-07,
      "logits/chosen": -2.3786637783050537,
      "logits/rejected": -2.4825186729431152,
      "logps/chosen": -259.32293701171875,
      "logps/rejected": -311.5961608886719,
      "loss": 0.1616,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7319447994232178,
      "rewards/margins": 2.7230427265167236,
      "rewards/rejected": -3.4549872875213623,
      "step": 1920
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.287896592244418e-07,
      "logits/chosen": -2.6222541332244873,
      "logits/rejected": -2.7417073249816895,
      "logps/chosen": -289.4549865722656,
      "logps/rejected": -323.2145080566406,
      "loss": 0.2004,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8005391955375671,
      "rewards/margins": 2.578702449798584,
      "rewards/rejected": -3.379241943359375,
      "step": 1930
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.2761457109283197e-07,
      "logits/chosen": -2.548220157623291,
      "logits/rejected": -2.6406500339508057,
      "logps/chosen": -261.72576904296875,
      "logps/rejected": -295.5674133300781,
      "loss": 0.2025,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7301163077354431,
      "rewards/margins": 2.5549705028533936,
      "rewards/rejected": -3.2850868701934814,
      "step": 1940
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.2643948296122207e-07,
      "logits/chosen": -2.4774460792541504,
      "logits/rejected": -2.657991886138916,
      "logps/chosen": -267.84100341796875,
      "logps/rejected": -342.40460205078125,
      "loss": 0.1662,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.629109799861908,
      "rewards/margins": 2.7617831230163574,
      "rewards/rejected": -3.3908932209014893,
      "step": 1950
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.252643948296122e-07,
      "logits/chosen": -2.5839192867279053,
      "logits/rejected": -2.6860103607177734,
      "logps/chosen": -249.51364135742188,
      "logps/rejected": -317.73211669921875,
      "loss": 0.2472,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6759640574455261,
      "rewards/margins": 2.3320960998535156,
      "rewards/rejected": -3.0080604553222656,
      "step": 1960
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.2408930669800233e-07,
      "logits/chosen": -2.491368293762207,
      "logits/rejected": -2.6284985542297363,
      "logps/chosen": -230.32162475585938,
      "logps/rejected": -321.59381103515625,
      "loss": 0.1595,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7740252017974854,
      "rewards/margins": 2.757596969604492,
      "rewards/rejected": -3.5316219329833984,
      "step": 1970
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.2291421856639243e-07,
      "logits/chosen": -2.5841212272644043,
      "logits/rejected": -2.7106974124908447,
      "logps/chosen": -254.65286254882812,
      "logps/rejected": -308.14300537109375,
      "loss": 0.1794,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6395988464355469,
      "rewards/margins": 2.817680835723877,
      "rewards/rejected": -3.457279682159424,
      "step": 1980
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.217391304347826e-07,
      "logits/chosen": -2.5222136974334717,
      "logits/rejected": -2.661198377609253,
      "logps/chosen": -271.9599304199219,
      "logps/rejected": -345.374267578125,
      "loss": 0.1475,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8032444715499878,
      "rewards/margins": 3.011528491973877,
      "rewards/rejected": -3.8147730827331543,
      "step": 1990
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.2056404230317274e-07,
      "logits/chosen": -2.690748691558838,
      "logits/rejected": -2.749650001525879,
      "logps/chosen": -317.8114013671875,
      "logps/rejected": -338.3028869628906,
      "loss": 0.271,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6004578471183777,
      "rewards/margins": 2.175527572631836,
      "rewards/rejected": -2.7759852409362793,
      "step": 2000
    },
    {
      "epoch": 1.27,
      "eval_logits/chosen": -2.630502462387085,
      "eval_logits/rejected": -2.757622241973877,
      "eval_logps/chosen": -278.3594665527344,
      "eval_logps/rejected": -320.97943115234375,
      "eval_loss": 0.1958618015050888,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.7446615695953369,
      "eval_rewards/margins": 2.682506561279297,
      "eval_rewards/rejected": -3.427168130874634,
      "eval_runtime": 106.2511,
      "eval_samples_per_second": 2.428,
      "eval_steps_per_second": 0.612,
      "step": 2000
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.1938895417156284e-07,
      "logits/chosen": -2.5724167823791504,
      "logits/rejected": -2.656190872192383,
      "logps/chosen": -256.85040283203125,
      "logps/rejected": -304.59619140625,
      "loss": 0.1931,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6460207104682922,
      "rewards/margins": 2.570005178451538,
      "rewards/rejected": -3.2160255908966064,
      "step": 2010
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.18213866039953e-07,
      "logits/chosen": -2.63395094871521,
      "logits/rejected": -2.7497212886810303,
      "logps/chosen": -242.5824737548828,
      "logps/rejected": -319.7347106933594,
      "loss": 0.2341,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.693947434425354,
      "rewards/margins": 2.399731397628784,
      "rewards/rejected": -3.0936789512634277,
      "step": 2020
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.170387779083431e-07,
      "logits/chosen": -2.482637882232666,
      "logits/rejected": -2.650730848312378,
      "logps/chosen": -256.2622985839844,
      "logps/rejected": -365.1991271972656,
      "loss": 0.1456,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7417194843292236,
      "rewards/margins": 3.259665012359619,
      "rewards/rejected": -4.001384258270264,
      "step": 2030
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.1586368977673325e-07,
      "logits/chosen": -2.492466926574707,
      "logits/rejected": -2.6561615467071533,
      "logps/chosen": -254.0111083984375,
      "logps/rejected": -331.40716552734375,
      "loss": 0.1412,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8182563781738281,
      "rewards/margins": 2.951646327972412,
      "rewards/rejected": -3.7699027061462402,
      "step": 2040
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.146886016451234e-07,
      "logits/chosen": -2.510547161102295,
      "logits/rejected": -2.662975311279297,
      "logps/chosen": -284.5359191894531,
      "logps/rejected": -345.28204345703125,
      "loss": 0.152,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6996340155601501,
      "rewards/margins": 2.8747878074645996,
      "rewards/rejected": -3.5744214057922363,
      "step": 2050
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.135135135135135e-07,
      "logits/chosen": -2.592583179473877,
      "logits/rejected": -2.715935468673706,
      "logps/chosen": -268.8671875,
      "logps/rejected": -334.01202392578125,
      "loss": 0.2093,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.6234585046768188,
      "rewards/margins": 2.7376465797424316,
      "rewards/rejected": -3.361105442047119,
      "step": 2060
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.1233842538190366e-07,
      "logits/chosen": -2.528543472290039,
      "logits/rejected": -2.6485679149627686,
      "logps/chosen": -261.3565673828125,
      "logps/rejected": -326.689697265625,
      "loss": 0.192,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8170403242111206,
      "rewards/margins": 2.712480306625366,
      "rewards/rejected": -3.5295207500457764,
      "step": 2070
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.1116333725029376e-07,
      "logits/chosen": -2.474642753601074,
      "logits/rejected": -2.603102207183838,
      "logps/chosen": -216.60116577148438,
      "logps/rejected": -295.6168212890625,
      "loss": 0.166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7863895893096924,
      "rewards/margins": 2.943730115890503,
      "rewards/rejected": -3.7301197052001953,
      "step": 2080
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.0998824911868386e-07,
      "logits/chosen": -2.582334041595459,
      "logits/rejected": -2.676194667816162,
      "logps/chosen": -250.197998046875,
      "logps/rejected": -311.15008544921875,
      "loss": 0.2109,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.770703911781311,
      "rewards/margins": 2.467026948928833,
      "rewards/rejected": -3.2377307415008545,
      "step": 2090
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.08813160987074e-07,
      "logits/chosen": -2.449924945831299,
      "logits/rejected": -2.6376631259918213,
      "logps/chosen": -253.0041961669922,
      "logps/rejected": -349.2939758300781,
      "loss": 0.127,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7250311374664307,
      "rewards/margins": 3.292039394378662,
      "rewards/rejected": -4.01707124710083,
      "step": 2100
    },
    {
      "epoch": 1.33,
      "eval_logits/chosen": -2.630169630050659,
      "eval_logits/rejected": -2.7571499347686768,
      "eval_logps/chosen": -278.5782165527344,
      "eval_logps/rejected": -321.84490966796875,
      "eval_loss": 0.19303728640079498,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.7665351033210754,
      "eval_rewards/margins": 2.747183322906494,
      "eval_rewards/rejected": -3.5137181282043457,
      "eval_runtime": 105.8846,
      "eval_samples_per_second": 2.437,
      "eval_steps_per_second": 0.614,
      "step": 2100
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.0763807285546417e-07,
      "logits/chosen": -2.509153366088867,
      "logits/rejected": -2.6735129356384277,
      "logps/chosen": -241.18399047851562,
      "logps/rejected": -334.8358154296875,
      "loss": 0.173,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.5902260541915894,
      "rewards/margins": 2.765799045562744,
      "rewards/rejected": -3.356025218963623,
      "step": 2110
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0646298472385427e-07,
      "logits/chosen": -2.500276565551758,
      "logits/rejected": -2.6179862022399902,
      "logps/chosen": -258.39422607421875,
      "logps/rejected": -317.87957763671875,
      "loss": 0.1775,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7223793864250183,
      "rewards/margins": 2.759366035461426,
      "rewards/rejected": -3.481745481491089,
      "step": 2120
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.052878965922444e-07,
      "logits/chosen": -2.6060385704040527,
      "logits/rejected": -2.7445321083068848,
      "logps/chosen": -192.28973388671875,
      "logps/rejected": -279.2763366699219,
      "loss": 0.2441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7277206778526306,
      "rewards/margins": 2.592848300933838,
      "rewards/rejected": -3.3205692768096924,
      "step": 2130
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.041128084606345e-07,
      "logits/chosen": -2.554877281188965,
      "logits/rejected": -2.6798787117004395,
      "logps/chosen": -271.58868408203125,
      "logps/rejected": -314.89508056640625,
      "loss": 0.1478,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7086837291717529,
      "rewards/margins": 3.083667278289795,
      "rewards/rejected": -3.792351245880127,
      "step": 2140
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0293772032902463e-07,
      "logits/chosen": -2.549607515335083,
      "logits/rejected": -2.702725887298584,
      "logps/chosen": -267.0545654296875,
      "logps/rejected": -356.22711181640625,
      "loss": 0.1737,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7205919623374939,
      "rewards/margins": 2.9332523345947266,
      "rewards/rejected": -3.6538443565368652,
      "step": 2150
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.017626321974148e-07,
      "logits/chosen": -2.562714099884033,
      "logits/rejected": -2.6540067195892334,
      "logps/chosen": -280.6510925292969,
      "logps/rejected": -335.317626953125,
      "loss": 0.1621,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8371556401252747,
      "rewards/margins": 2.977365255355835,
      "rewards/rejected": -3.814521074295044,
      "step": 2160
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.0058754406580494e-07,
      "logits/chosen": -2.408186674118042,
      "logits/rejected": -2.5006721019744873,
      "logps/chosen": -278.7369384765625,
      "logps/rejected": -317.28057861328125,
      "loss": 0.1759,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8426935076713562,
      "rewards/margins": 3.0238840579986572,
      "rewards/rejected": -3.866577625274658,
      "step": 2170
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.9941245593419504e-07,
      "logits/chosen": -2.4845364093780518,
      "logits/rejected": -2.6014716625213623,
      "logps/chosen": -266.1308898925781,
      "logps/rejected": -313.4009704589844,
      "loss": 0.1365,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8251141309738159,
      "rewards/margins": 2.811070442199707,
      "rewards/rejected": -3.6361846923828125,
      "step": 2180
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.982373678025852e-07,
      "logits/chosen": -2.547377824783325,
      "logits/rejected": -2.6679956912994385,
      "logps/chosen": -241.21044921875,
      "logps/rejected": -317.86749267578125,
      "loss": 0.1967,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7448156476020813,
      "rewards/margins": 2.8144774436950684,
      "rewards/rejected": -3.559293270111084,
      "step": 2190
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.970622796709753e-07,
      "logits/chosen": -2.6036388874053955,
      "logits/rejected": -2.7063612937927246,
      "logps/chosen": -268.3057861328125,
      "logps/rejected": -301.1413269042969,
      "loss": 0.2107,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8422848582267761,
      "rewards/margins": 2.62995982170105,
      "rewards/rejected": -3.4722447395324707,
      "step": 2200
    },
    {
      "epoch": 1.4,
      "eval_logits/chosen": -2.6304776668548584,
      "eval_logits/rejected": -2.7571909427642822,
      "eval_logps/chosen": -278.742919921875,
      "eval_logps/rejected": -322.590576171875,
      "eval_loss": 0.19047664105892181,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.783005952835083,
      "eval_rewards/margins": 2.805278778076172,
      "eval_rewards/rejected": -3.588284492492676,
      "eval_runtime": 103.8415,
      "eval_samples_per_second": 2.485,
      "eval_steps_per_second": 0.626,
      "step": 2200
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9588719153936545e-07,
      "logits/chosen": -2.5274107456207275,
      "logits/rejected": -2.649134635925293,
      "logps/chosen": -240.5630645751953,
      "logps/rejected": -309.73919677734375,
      "loss": 0.1927,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8205850720405579,
      "rewards/margins": 2.753375768661499,
      "rewards/rejected": -3.573960542678833,
      "step": 2210
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9471210340775555e-07,
      "logits/chosen": -2.5958077907562256,
      "logits/rejected": -2.730612277984619,
      "logps/chosen": -266.47467041015625,
      "logps/rejected": -338.05609130859375,
      "loss": 0.2069,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6244163513183594,
      "rewards/margins": 2.716456174850464,
      "rewards/rejected": -3.3408730030059814,
      "step": 2220
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.935370152761457e-07,
      "logits/chosen": -2.5137381553649902,
      "logits/rejected": -2.6499030590057373,
      "logps/chosen": -270.5130615234375,
      "logps/rejected": -333.3818664550781,
      "loss": 0.1855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8005086779594421,
      "rewards/margins": 2.7467246055603027,
      "rewards/rejected": -3.5472331047058105,
      "step": 2230
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.9236192714453586e-07,
      "logits/chosen": -2.5283362865448,
      "logits/rejected": -2.6609487533569336,
      "logps/chosen": -284.93585205078125,
      "logps/rejected": -346.6548767089844,
      "loss": 0.1718,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8945695161819458,
      "rewards/margins": 2.808443546295166,
      "rewards/rejected": -3.7030129432678223,
      "step": 2240
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.9118683901292596e-07,
      "logits/chosen": -2.5109972953796387,
      "logits/rejected": -2.6457619667053223,
      "logps/chosen": -259.888671875,
      "logps/rejected": -332.17840576171875,
      "loss": 0.1299,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8178337216377258,
      "rewards/margins": 3.28623628616333,
      "rewards/rejected": -4.10407018661499,
      "step": 2250
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.9001175088131606e-07,
      "logits/chosen": -2.525270938873291,
      "logits/rejected": -2.659254789352417,
      "logps/chosen": -305.78045654296875,
      "logps/rejected": -355.6806945800781,
      "loss": 0.1617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8407790064811707,
      "rewards/margins": 2.8437297344207764,
      "rewards/rejected": -3.68450927734375,
      "step": 2260
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.888366627497062e-07,
      "logits/chosen": -2.6310415267944336,
      "logits/rejected": -2.748731851577759,
      "logps/chosen": -273.66656494140625,
      "logps/rejected": -328.20758056640625,
      "loss": 0.2207,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7122267484664917,
      "rewards/margins": 2.7056260108947754,
      "rewards/rejected": -3.4178528785705566,
      "step": 2270
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8766157461809637e-07,
      "logits/chosen": -2.6047661304473877,
      "logits/rejected": -2.7593467235565186,
      "logps/chosen": -262.8388366699219,
      "logps/rejected": -339.60040283203125,
      "loss": 0.1537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8701519966125488,
      "rewards/margins": 2.992795705795288,
      "rewards/rejected": -3.862947940826416,
      "step": 2280
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8648648648648647e-07,
      "logits/chosen": -2.4941864013671875,
      "logits/rejected": -2.6116597652435303,
      "logps/chosen": -219.64794921875,
      "logps/rejected": -304.8808288574219,
      "loss": 0.1524,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9582231640815735,
      "rewards/margins": 3.0672593116760254,
      "rewards/rejected": -4.025482654571533,
      "step": 2290
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.853113983548766e-07,
      "logits/chosen": -2.645153522491455,
      "logits/rejected": -2.7601101398468018,
      "logps/chosen": -252.10574340820312,
      "logps/rejected": -314.9671325683594,
      "loss": 0.1977,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8973156213760376,
      "rewards/margins": 2.6417999267578125,
      "rewards/rejected": -3.5391154289245605,
      "step": 2300
    },
    {
      "epoch": 1.46,
      "eval_logits/chosen": -2.6300203800201416,
      "eval_logits/rejected": -2.756578207015991,
      "eval_logps/chosen": -278.89910888671875,
      "eval_logps/rejected": -323.28216552734375,
      "eval_loss": 0.18826447427272797,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.798626720905304,
      "eval_rewards/margins": 2.858816385269165,
      "eval_rewards/rejected": -3.657442569732666,
      "eval_runtime": 109.3707,
      "eval_samples_per_second": 2.359,
      "eval_steps_per_second": 0.594,
      "step": 2300
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.841363102232667e-07,
      "logits/chosen": -2.529174327850342,
      "logits/rejected": -2.6041061878204346,
      "logps/chosen": -232.40249633789062,
      "logps/rejected": -282.9012756347656,
      "loss": 0.2437,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.844650387763977,
      "rewards/margins": 2.499372720718384,
      "rewards/rejected": -3.3440232276916504,
      "step": 2310
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.829612220916568e-07,
      "logits/chosen": -2.406249761581421,
      "logits/rejected": -2.524104356765747,
      "logps/chosen": -269.17681884765625,
      "logps/rejected": -340.2377014160156,
      "loss": 0.1859,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.635347843170166,
      "rewards/margins": 2.9516003131866455,
      "rewards/rejected": -3.5869483947753906,
      "step": 2320
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.81786133960047e-07,
      "logits/chosen": -2.552602529525757,
      "logits/rejected": -2.6674323081970215,
      "logps/chosen": -273.44512939453125,
      "logps/rejected": -299.64300537109375,
      "loss": 0.1556,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8552175760269165,
      "rewards/margins": 2.986345052719116,
      "rewards/rejected": -3.8415629863739014,
      "step": 2330
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.8061104582843713e-07,
      "logits/chosen": -2.5263190269470215,
      "logits/rejected": -2.6461310386657715,
      "logps/chosen": -228.0097198486328,
      "logps/rejected": -310.5498046875,
      "loss": 0.1753,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7228946089744568,
      "rewards/margins": 2.9119372367858887,
      "rewards/rejected": -3.6348319053649902,
      "step": 2340
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.794359576968273e-07,
      "logits/chosen": -2.5303292274475098,
      "logits/rejected": -2.665847063064575,
      "logps/chosen": -257.12921142578125,
      "logps/rejected": -328.8788757324219,
      "loss": 0.1805,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7789188623428345,
      "rewards/margins": 2.8884904384613037,
      "rewards/rejected": -3.6674094200134277,
      "step": 2350
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.782608695652174e-07,
      "logits/chosen": -2.515049457550049,
      "logits/rejected": -2.6471707820892334,
      "logps/chosen": -261.67333984375,
      "logps/rejected": -329.1468200683594,
      "loss": 0.1626,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.779009222984314,
      "rewards/margins": 3.07348895072937,
      "rewards/rejected": -3.8524978160858154,
      "step": 2360
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.770857814336075e-07,
      "logits/chosen": -2.611295223236084,
      "logits/rejected": -2.6940250396728516,
      "logps/chosen": -295.1195373535156,
      "logps/rejected": -321.360107421875,
      "loss": 0.1956,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9063008427619934,
      "rewards/margins": 2.702629804611206,
      "rewards/rejected": -3.6089305877685547,
      "step": 2370
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.7591069330199765e-07,
      "logits/chosen": -2.521256923675537,
      "logits/rejected": -2.6360769271850586,
      "logps/chosen": -245.63583374023438,
      "logps/rejected": -296.31121826171875,
      "loss": 0.1991,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7459282279014587,
      "rewards/margins": 2.7573018074035645,
      "rewards/rejected": -3.503230333328247,
      "step": 2380
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.7473560517038775e-07,
      "logits/chosen": -2.4974143505096436,
      "logits/rejected": -2.662416934967041,
      "logps/chosen": -233.3050994873047,
      "logps/rejected": -334.03509521484375,
      "loss": 0.1359,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8811646699905396,
      "rewards/margins": 3.469583034515381,
      "rewards/rejected": -4.350748062133789,
      "step": 2390
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.735605170387779e-07,
      "logits/chosen": -2.430075168609619,
      "logits/rejected": -2.5878586769104004,
      "logps/chosen": -199.21080017089844,
      "logps/rejected": -302.89678955078125,
      "loss": 0.1655,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.764624297618866,
      "rewards/margins": 3.0026872158050537,
      "rewards/rejected": -3.7673118114471436,
      "step": 2400
    },
    {
      "epoch": 1.52,
      "eval_logits/chosen": -2.6289331912994385,
      "eval_logits/rejected": -2.755269765853882,
      "eval_logps/chosen": -279.1161193847656,
      "eval_logps/rejected": -323.8572082519531,
      "eval_loss": 0.187211811542511,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.8203264474868774,
      "eval_rewards/margins": 2.8946173191070557,
      "eval_rewards/rejected": -3.7149438858032227,
      "eval_runtime": 108.5173,
      "eval_samples_per_second": 2.378,
      "eval_steps_per_second": 0.599,
      "step": 2400
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.7238542890716806e-07,
      "logits/chosen": -2.5323901176452637,
      "logits/rejected": -2.6612801551818848,
      "logps/chosen": -255.3477783203125,
      "logps/rejected": -318.0188903808594,
      "loss": 0.1897,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8261678814888,
      "rewards/margins": 2.8823084831237793,
      "rewards/rejected": -3.7084763050079346,
      "step": 2410
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.7121034077555816e-07,
      "logits/chosen": -2.6246652603149414,
      "logits/rejected": -2.7231967449188232,
      "logps/chosen": -242.506103515625,
      "logps/rejected": -293.93145751953125,
      "loss": 0.2196,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7345197796821594,
      "rewards/margins": 2.6025593280792236,
      "rewards/rejected": -3.337078809738159,
      "step": 2420
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.7003525264394826e-07,
      "logits/chosen": -2.5028672218322754,
      "logits/rejected": -2.5940072536468506,
      "logps/chosen": -314.1603088378906,
      "logps/rejected": -362.56622314453125,
      "loss": 0.1744,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8953266143798828,
      "rewards/margins": 2.9029977321624756,
      "rewards/rejected": -3.7983245849609375,
      "step": 2430
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.688601645123384e-07,
      "logits/chosen": -2.6084141731262207,
      "logits/rejected": -2.739468574523926,
      "logps/chosen": -255.891845703125,
      "logps/rejected": -341.25347900390625,
      "loss": 0.1761,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.818018913269043,
      "rewards/margins": 3.0222034454345703,
      "rewards/rejected": -3.840222120285034,
      "step": 2440
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.6768507638072857e-07,
      "logits/chosen": -2.455069065093994,
      "logits/rejected": -2.612029552459717,
      "logps/chosen": -247.2393035888672,
      "logps/rejected": -333.34210205078125,
      "loss": 0.1494,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.843767523765564,
      "rewards/margins": 3.0948128700256348,
      "rewards/rejected": -3.938580274581909,
      "step": 2450
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.6650998824911867e-07,
      "logits/chosen": -2.4518589973449707,
      "logits/rejected": -2.622222423553467,
      "logps/chosen": -239.74856567382812,
      "logps/rejected": -337.02044677734375,
      "loss": 0.147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7651877999305725,
      "rewards/margins": 3.3721840381622314,
      "rewards/rejected": -4.137372016906738,
      "step": 2460
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.653349001175088e-07,
      "logits/chosen": -2.644845485687256,
      "logits/rejected": -2.7383341789245605,
      "logps/chosen": -311.63665771484375,
      "logps/rejected": -350.25738525390625,
      "loss": 0.1663,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9192525148391724,
      "rewards/margins": 3.0250587463378906,
      "rewards/rejected": -3.9443106651306152,
      "step": 2470
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.641598119858989e-07,
      "logits/chosen": -2.6303822994232178,
      "logits/rejected": -2.7424447536468506,
      "logps/chosen": -283.89752197265625,
      "logps/rejected": -332.1003112792969,
      "loss": 0.203,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.6722754240036011,
      "rewards/margins": 2.647305965423584,
      "rewards/rejected": -3.3195815086364746,
      "step": 2480
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.62984723854289e-07,
      "logits/chosen": -2.5485548973083496,
      "logits/rejected": -2.6848082542419434,
      "logps/chosen": -252.0126495361328,
      "logps/rejected": -325.1946716308594,
      "loss": 0.1802,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7533404231071472,
      "rewards/margins": 3.042879819869995,
      "rewards/rejected": -3.796220064163208,
      "step": 2490
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.618096357226792e-07,
      "logits/chosen": -2.555964231491089,
      "logits/rejected": -2.6456565856933594,
      "logps/chosen": -276.0560607910156,
      "logps/rejected": -316.6253967285156,
      "loss": 0.1776,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9099623560905457,
      "rewards/margins": 2.9367194175720215,
      "rewards/rejected": -3.846682071685791,
      "step": 2500
    },
    {
      "epoch": 1.59,
      "eval_logits/chosen": -2.628546953201294,
      "eval_logits/rejected": -2.75478196144104,
      "eval_logps/chosen": -279.3517761230469,
      "eval_logps/rejected": -324.5884704589844,
      "eval_loss": 0.18499596416950226,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.8438927531242371,
      "eval_rewards/margins": 2.9441792964935303,
      "eval_rewards/rejected": -3.788072347640991,
      "eval_runtime": 106.1174,
      "eval_samples_per_second": 2.431,
      "eval_steps_per_second": 0.613,
      "step": 2500
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.6063454759106933e-07,
      "logits/chosen": -2.5005877017974854,
      "logits/rejected": -2.617666244506836,
      "logps/chosen": -240.30355834960938,
      "logps/rejected": -316.4272155761719,
      "loss": 0.1913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8828660249710083,
      "rewards/margins": 2.7669081687927246,
      "rewards/rejected": -3.6497745513916016,
      "step": 2510
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.594594594594595e-07,
      "logits/chosen": -2.5812270641326904,
      "logits/rejected": -2.7395853996276855,
      "logps/chosen": -229.217529296875,
      "logps/rejected": -321.4227600097656,
      "loss": 0.1649,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9551548957824707,
      "rewards/margins": 3.1731815338134766,
      "rewards/rejected": -4.128336429595947,
      "step": 2520
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.582843713278496e-07,
      "logits/chosen": -2.5817012786865234,
      "logits/rejected": -2.7428698539733887,
      "logps/chosen": -266.1415100097656,
      "logps/rejected": -341.3614807128906,
      "loss": 0.2188,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8710711598396301,
      "rewards/margins": 2.78570818901062,
      "rewards/rejected": -3.6567795276641846,
      "step": 2530
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.571092831962397e-07,
      "logits/chosen": -2.583261013031006,
      "logits/rejected": -2.7345428466796875,
      "logps/chosen": -249.9550323486328,
      "logps/rejected": -336.6512451171875,
      "loss": 0.1747,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8093741536140442,
      "rewards/margins": 3.2250430583953857,
      "rewards/rejected": -4.034417152404785,
      "step": 2540
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5593419506462984e-07,
      "logits/chosen": -2.4961540699005127,
      "logits/rejected": -2.6065893173217773,
      "logps/chosen": -253.32864379882812,
      "logps/rejected": -313.80279541015625,
      "loss": 0.1611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8271971940994263,
      "rewards/margins": 3.0656933784484863,
      "rewards/rejected": -3.892890453338623,
      "step": 2550
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5475910693301995e-07,
      "logits/chosen": -2.59485125541687,
      "logits/rejected": -2.7342000007629395,
      "logps/chosen": -238.9771728515625,
      "logps/rejected": -325.7047424316406,
      "loss": 0.168,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7526350021362305,
      "rewards/margins": 3.09977388381958,
      "rewards/rejected": -3.8524088859558105,
      "step": 2560
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.535840188014101e-07,
      "logits/chosen": -2.505953550338745,
      "logits/rejected": -2.6342825889587402,
      "logps/chosen": -269.83209228515625,
      "logps/rejected": -313.6872863769531,
      "loss": 0.1565,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8948203921318054,
      "rewards/margins": 3.042961835861206,
      "rewards/rejected": -3.9377822875976562,
      "step": 2570
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.5240893066980025e-07,
      "logits/chosen": -2.4974305629730225,
      "logits/rejected": -2.6214256286621094,
      "logps/chosen": -258.09185791015625,
      "logps/rejected": -337.11480712890625,
      "loss": 0.1797,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7558786869049072,
      "rewards/margins": 3.011416435241699,
      "rewards/rejected": -3.7672951221466064,
      "step": 2580
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.5123384253819036e-07,
      "logits/chosen": -2.5998518466949463,
      "logits/rejected": -2.734579563140869,
      "logps/chosen": -286.82806396484375,
      "logps/rejected": -344.70318603515625,
      "loss": 0.1703,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7922006249427795,
      "rewards/margins": 3.3100032806396484,
      "rewards/rejected": -4.102203369140625,
      "step": 2590
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.5005875440658046e-07,
      "logits/chosen": -2.483070135116577,
      "logits/rejected": -2.645174741744995,
      "logps/chosen": -228.5210418701172,
      "logps/rejected": -323.61859130859375,
      "loss": 0.1372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9703159332275391,
      "rewards/margins": 3.6412150859832764,
      "rewards/rejected": -4.611530780792236,
      "step": 2600
    },
    {
      "epoch": 1.65,
      "eval_logits/chosen": -2.628246307373047,
      "eval_logits/rejected": -2.754406690597534,
      "eval_logps/chosen": -279.46087646484375,
      "eval_logps/rejected": -324.9880065917969,
      "eval_loss": 0.18496301770210266,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.854802668094635,
      "eval_rewards/margins": 2.9732229709625244,
      "eval_rewards/rejected": -3.8280258178710938,
      "eval_runtime": 102.962,
      "eval_samples_per_second": 2.506,
      "eval_steps_per_second": 0.631,
      "step": 2600
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.488836662749706e-07,
      "logits/chosen": -2.633091449737549,
      "logits/rejected": -2.718661069869995,
      "logps/chosen": -253.24557495117188,
      "logps/rejected": -301.527587890625,
      "loss": 0.2074,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8883869051933289,
      "rewards/margins": 2.6026313304901123,
      "rewards/rejected": -3.491018295288086,
      "step": 2610
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.4770857814336077e-07,
      "logits/chosen": -2.5530738830566406,
      "logits/rejected": -2.6860861778259277,
      "logps/chosen": -257.2315368652344,
      "logps/rejected": -346.2184753417969,
      "loss": 0.1813,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8997677564620972,
      "rewards/margins": 3.19789981842041,
      "rewards/rejected": -4.097667694091797,
      "step": 2620
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.4653349001175087e-07,
      "logits/chosen": -2.4484615325927734,
      "logits/rejected": -2.584386110305786,
      "logps/chosen": -299.26239013671875,
      "logps/rejected": -359.08935546875,
      "loss": 0.1206,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9096674919128418,
      "rewards/margins": 3.6679420471191406,
      "rewards/rejected": -4.577609539031982,
      "step": 2630
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.45358401880141e-07,
      "logits/chosen": -2.6539692878723145,
      "logits/rejected": -2.734898567199707,
      "logps/chosen": -266.04278564453125,
      "logps/rejected": -309.494873046875,
      "loss": 0.2805,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7391911745071411,
      "rewards/margins": 2.4634995460510254,
      "rewards/rejected": -3.202690839767456,
      "step": 2640
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.441833137485311e-07,
      "logits/chosen": -2.6056594848632812,
      "logits/rejected": -2.712914228439331,
      "logps/chosen": -287.87139892578125,
      "logps/rejected": -349.43792724609375,
      "loss": 0.1974,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8238168954849243,
      "rewards/margins": 2.9891397953033447,
      "rewards/rejected": -3.8129570484161377,
      "step": 2650
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.430082256169212e-07,
      "logits/chosen": -2.5024285316467285,
      "logits/rejected": -2.663832664489746,
      "logps/chosen": -283.6806640625,
      "logps/rejected": -373.3841857910156,
      "loss": 0.1242,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.9422612190246582,
      "rewards/margins": 3.501821517944336,
      "rewards/rejected": -4.444082260131836,
      "step": 2660
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.418331374853114e-07,
      "logits/chosen": -2.5351643562316895,
      "logits/rejected": -2.6721370220184326,
      "logps/chosen": -250.0267791748047,
      "logps/rejected": -327.53570556640625,
      "loss": 0.1586,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7074249386787415,
      "rewards/margins": 3.470238447189331,
      "rewards/rejected": -4.177663326263428,
      "step": 2670
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.4065804935370153e-07,
      "logits/chosen": -2.5762407779693604,
      "logits/rejected": -2.6625776290893555,
      "logps/chosen": -256.8520812988281,
      "logps/rejected": -301.7460632324219,
      "loss": 0.2547,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8985103368759155,
      "rewards/margins": 2.4213500022888184,
      "rewards/rejected": -3.3198599815368652,
      "step": 2680
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3948296122209163e-07,
      "logits/chosen": -2.527073383331299,
      "logits/rejected": -2.6502461433410645,
      "logps/chosen": -258.55487060546875,
      "logps/rejected": -306.0155944824219,
      "loss": 0.1468,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7603987455368042,
      "rewards/margins": 2.985889434814453,
      "rewards/rejected": -3.7462878227233887,
      "step": 2690
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.383078730904818e-07,
      "logits/chosen": -2.4828972816467285,
      "logits/rejected": -2.6326024532318115,
      "logps/chosen": -265.2709655761719,
      "logps/rejected": -356.79833984375,
      "loss": 0.15,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6885781288146973,
      "rewards/margins": 3.4638028144836426,
      "rewards/rejected": -4.15238094329834,
      "step": 2700
    },
    {
      "epoch": 1.71,
      "eval_logits/chosen": -2.628284215927124,
      "eval_logits/rejected": -2.7543022632598877,
      "eval_logps/chosen": -279.6464538574219,
      "eval_logps/rejected": -325.5001220703125,
      "eval_loss": 0.1836400181055069,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.8733604550361633,
      "eval_rewards/margins": 3.005875825881958,
      "eval_rewards/rejected": -3.8792362213134766,
      "eval_runtime": 104.0356,
      "eval_samples_per_second": 2.48,
      "eval_steps_per_second": 0.625,
      "step": 2700
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.371327849588719e-07,
      "logits/chosen": -2.575434446334839,
      "logits/rejected": -2.7064707279205322,
      "logps/chosen": -268.24481201171875,
      "logps/rejected": -321.6498107910156,
      "loss": 0.1814,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7702011466026306,
      "rewards/margins": 2.9064137935638428,
      "rewards/rejected": -3.6766152381896973,
      "step": 2710
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.3595769682726202e-07,
      "logits/chosen": -2.4542386531829834,
      "logits/rejected": -2.5391852855682373,
      "logps/chosen": -281.894287109375,
      "logps/rejected": -315.2430114746094,
      "loss": 0.1557,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8828303217887878,
      "rewards/margins": 3.172297954559326,
      "rewards/rejected": -4.055128574371338,
      "step": 2720
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.3478260869565217e-07,
      "logits/chosen": -2.620605945587158,
      "logits/rejected": -2.7156920433044434,
      "logps/chosen": -242.24227905273438,
      "logps/rejected": -298.11151123046875,
      "loss": 0.1814,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9634410738945007,
      "rewards/margins": 2.888660430908203,
      "rewards/rejected": -3.8521010875701904,
      "step": 2730
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.336075205640423e-07,
      "logits/chosen": -2.5120842456817627,
      "logits/rejected": -2.6340465545654297,
      "logps/chosen": -246.1355438232422,
      "logps/rejected": -330.52435302734375,
      "loss": 0.2047,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6820292472839355,
      "rewards/margins": 3.182408094406128,
      "rewards/rejected": -3.8644375801086426,
      "step": 2740
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.3243243243243243e-07,
      "logits/chosen": -2.465364694595337,
      "logits/rejected": -2.6128716468811035,
      "logps/chosen": -233.1071319580078,
      "logps/rejected": -335.6039123535156,
      "loss": 0.1394,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8780118227005005,
      "rewards/margins": 3.4173004627227783,
      "rewards/rejected": -4.29531192779541,
      "step": 2750
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.3125734430082255e-07,
      "logits/chosen": -2.5143094062805176,
      "logits/rejected": -2.6679439544677734,
      "logps/chosen": -257.15130615234375,
      "logps/rejected": -353.59722900390625,
      "loss": 0.1512,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8592759966850281,
      "rewards/margins": 3.3945224285125732,
      "rewards/rejected": -4.253798484802246,
      "step": 2760
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.3008225616921268e-07,
      "logits/chosen": -2.5617387294769287,
      "logits/rejected": -2.7220163345336914,
      "logps/chosen": -318.2297058105469,
      "logps/rejected": -377.73773193359375,
      "loss": 0.1586,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9121103286743164,
      "rewards/margins": 3.2415764331817627,
      "rewards/rejected": -4.153687000274658,
      "step": 2770
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.289071680376028e-07,
      "logits/chosen": -2.4399588108062744,
      "logits/rejected": -2.612267017364502,
      "logps/chosen": -249.0401153564453,
      "logps/rejected": -351.3318786621094,
      "loss": 0.0925,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9835712313652039,
      "rewards/margins": 3.7314536571502686,
      "rewards/rejected": -4.715023994445801,
      "step": 2780
    },
    {
      "epoch": 1.77,
      "learning_rate": 2.2773207990599294e-07,
      "logits/chosen": -2.5065250396728516,
      "logits/rejected": -2.629993438720703,
      "logps/chosen": -254.3542022705078,
      "logps/rejected": -329.5284118652344,
      "loss": 0.1698,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8880063891410828,
      "rewards/margins": 3.5896058082580566,
      "rewards/rejected": -4.477612495422363,
      "step": 2790
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.2655699177438307e-07,
      "logits/chosen": -2.4954376220703125,
      "logits/rejected": -2.6363935470581055,
      "logps/chosen": -236.16311645507812,
      "logps/rejected": -328.7548522949219,
      "loss": 0.1338,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7927232980728149,
      "rewards/margins": 3.4046483039855957,
      "rewards/rejected": -4.197371482849121,
      "step": 2800
    },
    {
      "epoch": 1.78,
      "eval_logits/chosen": -2.6282033920288086,
      "eval_logits/rejected": -2.7541122436523438,
      "eval_logps/chosen": -279.64862060546875,
      "eval_logps/rejected": -325.83929443359375,
      "eval_loss": 0.1822524517774582,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.8735765814781189,
      "eval_rewards/margins": 3.0395803451538086,
      "eval_rewards/rejected": -3.913156747817993,
      "eval_runtime": 108.7545,
      "eval_samples_per_second": 2.372,
      "eval_steps_per_second": 0.598,
      "step": 2800
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.2538190364277322e-07,
      "logits/chosen": -2.542961597442627,
      "logits/rejected": -2.6782212257385254,
      "logps/chosen": -307.22650146484375,
      "logps/rejected": -365.54840087890625,
      "loss": 0.1392,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9173371195793152,
      "rewards/margins": 3.4531643390655518,
      "rewards/rejected": -4.3705010414123535,
      "step": 2810
    },
    {
      "epoch": 1.79,
      "learning_rate": 2.2420681551116332e-07,
      "logits/chosen": -2.415771245956421,
      "logits/rejected": -2.542311191558838,
      "logps/chosen": -255.05252075195312,
      "logps/rejected": -328.19866943359375,
      "loss": 0.1455,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7641615271568298,
      "rewards/margins": 3.332284927368164,
      "rewards/rejected": -4.0964460372924805,
      "step": 2820
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.2303172737955345e-07,
      "logits/chosen": -2.480813503265381,
      "logits/rejected": -2.5997071266174316,
      "logps/chosen": -234.4130401611328,
      "logps/rejected": -307.7239685058594,
      "loss": 0.1526,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9471356272697449,
      "rewards/margins": 3.2659358978271484,
      "rewards/rejected": -4.213071823120117,
      "step": 2830
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.218566392479436e-07,
      "logits/chosen": -2.518893241882324,
      "logits/rejected": -2.663313388824463,
      "logps/chosen": -224.6551971435547,
      "logps/rejected": -310.76348876953125,
      "loss": 0.1355,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7350938320159912,
      "rewards/margins": 3.3886139392852783,
      "rewards/rejected": -4.1237077713012695,
      "step": 2840
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.206815511163337e-07,
      "logits/chosen": -2.5833730697631836,
      "logits/rejected": -2.729428768157959,
      "logps/chosen": -239.071044921875,
      "logps/rejected": -309.70196533203125,
      "loss": 0.1633,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7405908107757568,
      "rewards/margins": 3.2603752613067627,
      "rewards/rejected": -4.0009660720825195,
      "step": 2850
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.1950646298472383e-07,
      "logits/chosen": -2.4613192081451416,
      "logits/rejected": -2.5912773609161377,
      "logps/chosen": -240.5667266845703,
      "logps/rejected": -323.4114990234375,
      "loss": 0.1087,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.098568320274353,
      "rewards/margins": 3.446556568145752,
      "rewards/rejected": -4.5451250076293945,
      "step": 2860
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.18331374853114e-07,
      "logits/chosen": -2.4373083114624023,
      "logits/rejected": -2.6097519397735596,
      "logps/chosen": -232.98495483398438,
      "logps/rejected": -334.99713134765625,
      "loss": 0.1194,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8801986575126648,
      "rewards/margins": 3.6822407245635986,
      "rewards/rejected": -4.56243896484375,
      "step": 2870
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.171562867215041e-07,
      "logits/chosen": -2.551697254180908,
      "logits/rejected": -2.7013208866119385,
      "logps/chosen": -220.43130493164062,
      "logps/rejected": -314.50238037109375,
      "loss": 0.1786,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9244735836982727,
      "rewards/margins": 3.1510705947875977,
      "rewards/rejected": -4.075544357299805,
      "step": 2880
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1598119858989422e-07,
      "logits/chosen": -2.5419392585754395,
      "logits/rejected": -2.6923465728759766,
      "logps/chosen": -275.2723083496094,
      "logps/rejected": -342.51800537109375,
      "loss": 0.1802,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8420646786689758,
      "rewards/margins": 3.0612165927886963,
      "rewards/rejected": -3.9032809734344482,
      "step": 2890
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.1480611045828437e-07,
      "logits/chosen": -2.4186792373657227,
      "logits/rejected": -2.577014923095703,
      "logps/chosen": -291.5527648925781,
      "logps/rejected": -351.9775085449219,
      "loss": 0.1507,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.0090060234069824,
      "rewards/margins": 3.3944592475891113,
      "rewards/rejected": -4.403465270996094,
      "step": 2900
    },
    {
      "epoch": 1.84,
      "eval_logits/chosen": -2.6273231506347656,
      "eval_logits/rejected": -2.7532639503479004,
      "eval_logps/chosen": -279.8443908691406,
      "eval_logps/rejected": -326.2652893066406,
      "eval_loss": 0.18107537925243378,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.8931529521942139,
      "eval_rewards/margins": 3.0626070499420166,
      "eval_rewards/rejected": -3.9557597637176514,
      "eval_runtime": 99.7978,
      "eval_samples_per_second": 2.585,
      "eval_steps_per_second": 0.651,
      "step": 2900
    },
    {
      "epoch": 1.85,
      "learning_rate": 2.136310223266745e-07,
      "logits/chosen": -2.4943299293518066,
      "logits/rejected": -2.6106464862823486,
      "logps/chosen": -292.67315673828125,
      "logps/rejected": -342.3676452636719,
      "loss": 0.1278,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8295286893844604,
      "rewards/margins": 3.140049457550049,
      "rewards/rejected": -3.969578266143799,
      "step": 2910
    },
    {
      "epoch": 1.85,
      "learning_rate": 2.124559341950646e-07,
      "logits/chosen": -2.6118836402893066,
      "logits/rejected": -2.690326452255249,
      "logps/chosen": -294.5302429199219,
      "logps/rejected": -317.48614501953125,
      "loss": 0.137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0399130582809448,
      "rewards/margins": 3.159194231033325,
      "rewards/rejected": -4.1991071701049805,
      "step": 2920
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.1128084606345475e-07,
      "logits/chosen": -2.583739757537842,
      "logits/rejected": -2.6843104362487793,
      "logps/chosen": -236.9778289794922,
      "logps/rejected": -298.92889404296875,
      "loss": 0.1642,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9768341779708862,
      "rewards/margins": 3.1713690757751465,
      "rewards/rejected": -4.148202896118164,
      "step": 2930
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.1010575793184488e-07,
      "logits/chosen": -2.5355803966522217,
      "logits/rejected": -2.6054155826568604,
      "logps/chosen": -307.74188232421875,
      "logps/rejected": -338.76483154296875,
      "loss": 0.1653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0456215143203735,
      "rewards/margins": 3.2320404052734375,
      "rewards/rejected": -4.2776618003845215,
      "step": 2940
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.08930669800235e-07,
      "logits/chosen": -2.5585834980010986,
      "logits/rejected": -2.6883463859558105,
      "logps/chosen": -337.6223449707031,
      "logps/rejected": -376.09259033203125,
      "loss": 0.1446,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9894275665283203,
      "rewards/margins": 3.3709633350372314,
      "rewards/rejected": -4.360390663146973,
      "step": 2950
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0775558166862514e-07,
      "logits/chosen": -2.6321702003479004,
      "logits/rejected": -2.717146396636963,
      "logps/chosen": -294.2631530761719,
      "logps/rejected": -345.49053955078125,
      "loss": 0.2244,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0043416023254395,
      "rewards/margins": 2.61612606048584,
      "rewards/rejected": -3.6204674243927,
      "step": 2960
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0658049353701526e-07,
      "logits/chosen": -2.5753731727600098,
      "logits/rejected": -2.7137269973754883,
      "logps/chosen": -221.07778930664062,
      "logps/rejected": -293.5127868652344,
      "loss": 0.1862,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8521106839179993,
      "rewards/margins": 3.088805913925171,
      "rewards/rejected": -3.9409167766571045,
      "step": 2970
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.0540540540540542e-07,
      "logits/chosen": -2.4867992401123047,
      "logits/rejected": -2.6059253215789795,
      "logps/chosen": -288.48614501953125,
      "logps/rejected": -340.2772521972656,
      "loss": 0.0969,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9175114631652832,
      "rewards/margins": 3.6409244537353516,
      "rewards/rejected": -4.558435916900635,
      "step": 2980
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0423031727379552e-07,
      "logits/chosen": -2.576115369796753,
      "logits/rejected": -2.7292776107788086,
      "logps/chosen": -240.19393920898438,
      "logps/rejected": -335.06793212890625,
      "loss": 0.1819,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8659070730209351,
      "rewards/margins": 3.258894443511963,
      "rewards/rejected": -4.1248016357421875,
      "step": 2990
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0305522914218565e-07,
      "logits/chosen": -2.511610507965088,
      "logits/rejected": -2.7145934104919434,
      "logps/chosen": -263.8073425292969,
      "logps/rejected": -363.18194580078125,
      "loss": 0.1615,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8913321495056152,
      "rewards/margins": 3.276223659515381,
      "rewards/rejected": -4.167555809020996,
      "step": 3000
    },
    {
      "epoch": 1.9,
      "eval_logits/chosen": -2.6274654865264893,
      "eval_logits/rejected": -2.7533252239227295,
      "eval_logps/chosen": -279.89923095703125,
      "eval_logps/rejected": -326.4981384277344,
      "eval_loss": 0.18113359808921814,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.8986391425132751,
      "eval_rewards/margins": 3.080401659011841,
      "eval_rewards/rejected": -3.97904109954834,
      "eval_runtime": 100.275,
      "eval_samples_per_second": 2.573,
      "eval_steps_per_second": 0.648,
      "step": 3000
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.018801410105758e-07,
      "logits/chosen": -2.6342387199401855,
      "logits/rejected": -2.734529733657837,
      "logps/chosen": -296.2749328613281,
      "logps/rejected": -349.5816345214844,
      "loss": 0.1314,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.768647313117981,
      "rewards/margins": 3.6035220623016357,
      "rewards/rejected": -4.37216854095459,
      "step": 3010
    },
    {
      "epoch": 1.92,
      "learning_rate": 2.007050528789659e-07,
      "logits/chosen": -2.3809564113616943,
      "logits/rejected": -2.568582057952881,
      "logps/chosen": -243.52249145507812,
      "logps/rejected": -332.52349853515625,
      "loss": 0.1344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6865008473396301,
      "rewards/margins": 3.5618433952331543,
      "rewards/rejected": -4.2483439445495605,
      "step": 3020
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.9952996474735603e-07,
      "logits/chosen": -2.55562686920166,
      "logits/rejected": -2.6805107593536377,
      "logps/chosen": -251.91543579101562,
      "logps/rejected": -315.0675048828125,
      "loss": 0.1611,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8510581254959106,
      "rewards/margins": 3.3322606086730957,
      "rewards/rejected": -4.183318138122559,
      "step": 3030
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.9835487661574619e-07,
      "logits/chosen": -2.4944286346435547,
      "logits/rejected": -2.6402807235717773,
      "logps/chosen": -285.4676818847656,
      "logps/rejected": -341.287353515625,
      "loss": 0.1005,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0004218816757202,
      "rewards/margins": 3.667121410369873,
      "rewards/rejected": -4.667543411254883,
      "step": 3040
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.971797884841363e-07,
      "logits/chosen": -2.534848690032959,
      "logits/rejected": -2.6611368656158447,
      "logps/chosen": -284.7879333496094,
      "logps/rejected": -350.4801025390625,
      "loss": 0.1609,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0162169933319092,
      "rewards/margins": 3.189302921295166,
      "rewards/rejected": -4.205519199371338,
      "step": 3050
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.9600470035252641e-07,
      "logits/chosen": -2.518596887588501,
      "logits/rejected": -2.63787841796875,
      "logps/chosen": -231.1822052001953,
      "logps/rejected": -309.16400146484375,
      "loss": 0.1758,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0202105045318604,
      "rewards/margins": 3.156257152557373,
      "rewards/rejected": -4.1764678955078125,
      "step": 3060
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.9482961222091657e-07,
      "logits/chosen": -2.6840717792510986,
      "logits/rejected": -2.792886257171631,
      "logps/chosen": -261.45904541015625,
      "logps/rejected": -313.09478759765625,
      "loss": 0.1987,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0175288915634155,
      "rewards/margins": 2.9839608669281006,
      "rewards/rejected": -4.001489639282227,
      "step": 3070
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.936545240893067e-07,
      "logits/chosen": -2.5590438842773438,
      "logits/rejected": -2.686429977416992,
      "logps/chosen": -282.2024230957031,
      "logps/rejected": -363.87176513671875,
      "loss": 0.1714,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8027384877204895,
      "rewards/margins": 3.4363608360290527,
      "rewards/rejected": -4.239099502563477,
      "step": 3080
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.924794359576968e-07,
      "logits/chosen": -2.5604755878448486,
      "logits/rejected": -2.7263646125793457,
      "logps/chosen": -284.63592529296875,
      "logps/rejected": -367.0308837890625,
      "loss": 0.2337,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.9264057278633118,
      "rewards/margins": 3.2333292961120605,
      "rewards/rejected": -4.159735202789307,
      "step": 3090
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.9130434782608695e-07,
      "logits/chosen": -2.538912534713745,
      "logits/rejected": -2.67553448677063,
      "logps/chosen": -292.20513916015625,
      "logps/rejected": -355.1044616699219,
      "loss": 0.1656,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9621899724006653,
      "rewards/margins": 3.504662275314331,
      "rewards/rejected": -4.46685266494751,
      "step": 3100
    },
    {
      "epoch": 1.97,
      "eval_logits/chosen": -2.6269664764404297,
      "eval_logits/rejected": -2.7527925968170166,
      "eval_logps/chosen": -279.9523010253906,
      "eval_logps/rejected": -326.7593688964844,
      "eval_loss": 0.18003487586975098,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.903947651386261,
      "eval_rewards/margins": 3.1012189388275146,
      "eval_rewards/rejected": -4.005166530609131,
      "eval_runtime": 99.2802,
      "eval_samples_per_second": 2.599,
      "eval_steps_per_second": 0.655,
      "step": 3100
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.9012925969447708e-07,
      "logits/chosen": -2.5181639194488525,
      "logits/rejected": -2.6622474193573,
      "logps/chosen": -219.726318359375,
      "logps/rejected": -311.7294616699219,
      "loss": 0.1495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7814009189605713,
      "rewards/margins": 3.4612441062927246,
      "rewards/rejected": -4.242645740509033,
      "step": 3110
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8895417156286718e-07,
      "logits/chosen": -2.5011534690856934,
      "logits/rejected": -2.6525678634643555,
      "logps/chosen": -248.6040802001953,
      "logps/rejected": -323.9615478515625,
      "loss": 0.1638,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9456720352172852,
      "rewards/margins": 3.2832443714141846,
      "rewards/rejected": -4.228917121887207,
      "step": 3120
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.8777908343125734e-07,
      "logits/chosen": -2.5089216232299805,
      "logits/rejected": -2.5820202827453613,
      "logps/chosen": -250.28482055664062,
      "logps/rejected": -278.0976867675781,
      "loss": 0.1928,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8614521026611328,
      "rewards/margins": 2.670563220977783,
      "rewards/rejected": -3.532015323638916,
      "step": 3130
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.8660399529964746e-07,
      "logits/chosen": -2.442479372024536,
      "logits/rejected": -2.5443613529205322,
      "logps/chosen": -245.1953887939453,
      "logps/rejected": -290.0235290527344,
      "loss": 0.2031,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8449136018753052,
      "rewards/margins": 2.9672536849975586,
      "rewards/rejected": -3.8121674060821533,
      "step": 3140
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8542890716803762e-07,
      "logits/chosen": -2.5265073776245117,
      "logits/rejected": -2.685546636581421,
      "logps/chosen": -261.3786315917969,
      "logps/rejected": -338.96295166015625,
      "loss": 0.1732,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9381521344184875,
      "rewards/margins": 3.295395612716675,
      "rewards/rejected": -4.233547687530518,
      "step": 3150
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.8425381903642772e-07,
      "logits/chosen": -2.5398635864257812,
      "logits/rejected": -2.7591240406036377,
      "logps/chosen": -238.38046264648438,
      "logps/rejected": -351.30242919921875,
      "loss": 0.13,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9397295117378235,
      "rewards/margins": 3.5519003868103027,
      "rewards/rejected": -4.491629600524902,
      "step": 3160
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.8307873090481785e-07,
      "logits/chosen": -2.550031900405884,
      "logits/rejected": -2.62827730178833,
      "logps/chosen": -299.3024597167969,
      "logps/rejected": -337.82781982421875,
      "loss": 0.1959,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.928261935710907,
      "rewards/margins": 3.03495454788208,
      "rewards/rejected": -3.9632163047790527,
      "step": 3170
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.81903642773208e-07,
      "logits/chosen": -2.479654312133789,
      "logits/rejected": -2.602837085723877,
      "logps/chosen": -262.5113830566406,
      "logps/rejected": -339.10760498046875,
      "loss": 0.1629,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.84858238697052,
      "rewards/margins": 3.2553043365478516,
      "rewards/rejected": -4.103886604309082,
      "step": 3180
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.807285546415981e-07,
      "logits/chosen": -2.5605947971343994,
      "logits/rejected": -2.6545703411102295,
      "logps/chosen": -266.50909423828125,
      "logps/rejected": -318.95648193359375,
      "loss": 0.1696,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8396397829055786,
      "rewards/margins": 3.124613046646118,
      "rewards/rejected": -3.9642529487609863,
      "step": 3190
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.7955346650998823e-07,
      "logits/chosen": -2.5116517543792725,
      "logits/rejected": -2.6574039459228516,
      "logps/chosen": -214.2110595703125,
      "logps/rejected": -321.4629211425781,
      "loss": 0.1398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9295800924301147,
      "rewards/margins": 3.4923148155212402,
      "rewards/rejected": -4.4218950271606445,
      "step": 3200
    },
    {
      "epoch": 2.03,
      "eval_logits/chosen": -2.6278295516967773,
      "eval_logits/rejected": -2.753410816192627,
      "eval_logps/chosen": -280.0360412597656,
      "eval_logps/rejected": -326.9659729003906,
      "eval_loss": 0.1797335296869278,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9123177528381348,
      "eval_rewards/margins": 3.1135058403015137,
      "eval_rewards/rejected": -4.025823593139648,
      "eval_runtime": 107.039,
      "eval_samples_per_second": 2.41,
      "eval_steps_per_second": 0.607,
      "step": 3200
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.7837837837837838e-07,
      "logits/chosen": -2.5115394592285156,
      "logits/rejected": -2.6251227855682373,
      "logps/chosen": -246.1479949951172,
      "logps/rejected": -338.1456604003906,
      "loss": 0.1484,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9279565811157227,
      "rewards/margins": 3.3821563720703125,
      "rewards/rejected": -4.310112476348877,
      "step": 3210
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.772032902467685e-07,
      "logits/chosen": -2.5599820613861084,
      "logits/rejected": -2.732851028442383,
      "logps/chosen": -222.70584106445312,
      "logps/rejected": -324.3047180175781,
      "loss": 0.2,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.871590793132782,
      "rewards/margins": 3.3388302326202393,
      "rewards/rejected": -4.210420608520508,
      "step": 3220
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.7602820211515861e-07,
      "logits/chosen": -2.6067464351654053,
      "logits/rejected": -2.681145191192627,
      "logps/chosen": -257.8177795410156,
      "logps/rejected": -305.45758056640625,
      "loss": 0.1817,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8875662684440613,
      "rewards/margins": 2.9419164657592773,
      "rewards/rejected": -3.8294830322265625,
      "step": 3230
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.7485311398354877e-07,
      "logits/chosen": -2.672916889190674,
      "logits/rejected": -2.7730250358581543,
      "logps/chosen": -253.174072265625,
      "logps/rejected": -313.11676025390625,
      "loss": 0.2329,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8167648315429688,
      "rewards/margins": 2.720130205154419,
      "rewards/rejected": -3.5368945598602295,
      "step": 3240
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.736780258519389e-07,
      "logits/chosen": -2.5312917232513428,
      "logits/rejected": -2.6694445610046387,
      "logps/chosen": -270.8564453125,
      "logps/rejected": -344.3938293457031,
      "loss": 0.1351,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9721826314926147,
      "rewards/margins": 3.610694408416748,
      "rewards/rejected": -4.582877159118652,
      "step": 3250
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.72502937720329e-07,
      "logits/chosen": -2.5625193119049072,
      "logits/rejected": -2.669475793838501,
      "logps/chosen": -252.27902221679688,
      "logps/rejected": -300.34375,
      "loss": 0.1719,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9256380200386047,
      "rewards/margins": 3.256152629852295,
      "rewards/rejected": -4.181790351867676,
      "step": 3260
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.7132784958871915e-07,
      "logits/chosen": -2.6262800693511963,
      "logits/rejected": -2.7180352210998535,
      "logps/chosen": -285.98583984375,
      "logps/rejected": -317.5250244140625,
      "loss": 0.1665,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.787746012210846,
      "rewards/margins": 3.023850679397583,
      "rewards/rejected": -3.811596632003784,
      "step": 3270
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.7015276145710928e-07,
      "logits/chosen": -2.6141204833984375,
      "logits/rejected": -2.725219964981079,
      "logps/chosen": -264.04156494140625,
      "logps/rejected": -332.28240966796875,
      "loss": 0.1595,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.156468391418457,
      "rewards/margins": 3.2472927570343018,
      "rewards/rejected": -4.403761386871338,
      "step": 3280
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.6897767332549938e-07,
      "logits/chosen": -2.5131583213806152,
      "logits/rejected": -2.637444257736206,
      "logps/chosen": -292.16864013671875,
      "logps/rejected": -332.4121398925781,
      "loss": 0.136,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8695871233940125,
      "rewards/margins": 3.3206393718719482,
      "rewards/rejected": -4.1902265548706055,
      "step": 3290
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.6780258519388953e-07,
      "logits/chosen": -2.545576810836792,
      "logits/rejected": -2.681370258331299,
      "logps/chosen": -351.9949645996094,
      "logps/rejected": -414.196044921875,
      "loss": 0.1929,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9536555409431458,
      "rewards/margins": 3.1913812160491943,
      "rewards/rejected": -4.145036697387695,
      "step": 3300
    },
    {
      "epoch": 2.09,
      "eval_logits/chosen": -2.626879930496216,
      "eval_logits/rejected": -2.752368211746216,
      "eval_logps/chosen": -280.01123046875,
      "eval_logps/rejected": -327.0879211425781,
      "eval_loss": 0.17915986478328705,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9098411202430725,
      "eval_rewards/margins": 3.128176212310791,
      "eval_rewards/rejected": -4.0380167961120605,
      "eval_runtime": 113.1653,
      "eval_samples_per_second": 2.28,
      "eval_steps_per_second": 0.574,
      "step": 3300
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.6662749706227966e-07,
      "logits/chosen": -2.4755101203918457,
      "logits/rejected": -2.5843305587768555,
      "logps/chosen": -276.5982666015625,
      "logps/rejected": -322.8077392578125,
      "loss": 0.177,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8209638595581055,
      "rewards/margins": 3.0441031455993652,
      "rewards/rejected": -3.8650670051574707,
      "step": 3310
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.654524089306698e-07,
      "logits/chosen": -2.6766388416290283,
      "logits/rejected": -2.742995500564575,
      "logps/chosen": -267.0733642578125,
      "logps/rejected": -302.3008728027344,
      "loss": 0.2051,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9289132952690125,
      "rewards/margins": 2.661316156387329,
      "rewards/rejected": -3.590229034423828,
      "step": 3320
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.6427732079905992e-07,
      "logits/chosen": -2.49891996383667,
      "logits/rejected": -2.594839334487915,
      "logps/chosen": -212.095947265625,
      "logps/rejected": -279.9945068359375,
      "loss": 0.1629,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8233148455619812,
      "rewards/margins": 3.2924067974090576,
      "rewards/rejected": -4.115721702575684,
      "step": 3330
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6310223266745005e-07,
      "logits/chosen": -2.5815701484680176,
      "logits/rejected": -2.6818337440490723,
      "logps/chosen": -311.83197021484375,
      "logps/rejected": -342.59161376953125,
      "loss": 0.1389,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.04054856300354,
      "rewards/margins": 3.3212532997131348,
      "rewards/rejected": -4.361802101135254,
      "step": 3340
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.619271445358402e-07,
      "logits/chosen": -2.494476079940796,
      "logits/rejected": -2.588510036468506,
      "logps/chosen": -255.8999481201172,
      "logps/rejected": -306.3526306152344,
      "loss": 0.1685,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8534440994262695,
      "rewards/margins": 3.090205192565918,
      "rewards/rejected": -3.9436492919921875,
      "step": 3350
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.607520564042303e-07,
      "logits/chosen": -2.514962673187256,
      "logits/rejected": -2.6802875995635986,
      "logps/chosen": -196.5522918701172,
      "logps/rejected": -290.5397644042969,
      "loss": 0.1675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8389670252799988,
      "rewards/margins": 3.19274640083313,
      "rewards/rejected": -4.031713485717773,
      "step": 3360
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5957696827262043e-07,
      "logits/chosen": -2.461728572845459,
      "logits/rejected": -2.617161989212036,
      "logps/chosen": -231.8739776611328,
      "logps/rejected": -334.3666076660156,
      "loss": 0.1277,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8081679344177246,
      "rewards/margins": 3.482015609741211,
      "rewards/rejected": -4.290183067321777,
      "step": 3370
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5840188014101058e-07,
      "logits/chosen": -2.569915771484375,
      "logits/rejected": -2.7211246490478516,
      "logps/chosen": -254.9673614501953,
      "logps/rejected": -344.7939147949219,
      "loss": 0.1629,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.80149906873703,
      "rewards/margins": 3.3813891410827637,
      "rewards/rejected": -4.182888031005859,
      "step": 3380
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.572267920094007e-07,
      "logits/chosen": -2.540274143218994,
      "logits/rejected": -2.650144100189209,
      "logps/chosen": -273.9688415527344,
      "logps/rejected": -340.51214599609375,
      "loss": 0.1824,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9421696662902832,
      "rewards/margins": 3.191805124282837,
      "rewards/rejected": -4.133975028991699,
      "step": 3390
    },
    {
      "epoch": 2.16,
      "learning_rate": 1.560517038777908e-07,
      "logits/chosen": -2.546204090118408,
      "logits/rejected": -2.693211078643799,
      "logps/chosen": -280.6536560058594,
      "logps/rejected": -356.1172180175781,
      "loss": 0.1616,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9629494547843933,
      "rewards/margins": 3.3437588214874268,
      "rewards/rejected": -4.306708335876465,
      "step": 3400
    },
    {
      "epoch": 2.16,
      "eval_logits/chosen": -2.626338243484497,
      "eval_logits/rejected": -2.7519478797912598,
      "eval_logps/chosen": -280.1615905761719,
      "eval_logps/rejected": -327.3301086425781,
      "eval_loss": 0.1787494421005249,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.92487633228302,
      "eval_rewards/margins": 3.137359142303467,
      "eval_rewards/rejected": -4.0622358322143555,
      "eval_runtime": 103.8581,
      "eval_samples_per_second": 2.484,
      "eval_steps_per_second": 0.626,
      "step": 3400
    },
    {
      "epoch": 2.16,
      "learning_rate": 1.5487661574618097e-07,
      "logits/chosen": -2.4918081760406494,
      "logits/rejected": -2.644871950149536,
      "logps/chosen": -262.69305419921875,
      "logps/rejected": -337.6416320800781,
      "loss": 0.1614,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8792058825492859,
      "rewards/margins": 3.2974154949188232,
      "rewards/rejected": -4.176621437072754,
      "step": 3410
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.537015276145711e-07,
      "logits/chosen": -2.503817081451416,
      "logits/rejected": -2.591966152191162,
      "logps/chosen": -294.28857421875,
      "logps/rejected": -341.55633544921875,
      "loss": 0.2135,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7600934505462646,
      "rewards/margins": 2.8195152282714844,
      "rewards/rejected": -3.579608917236328,
      "step": 3420
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.525264394829612e-07,
      "logits/chosen": -2.440645694732666,
      "logits/rejected": -2.5400350093841553,
      "logps/chosen": -273.10931396484375,
      "logps/rejected": -355.63800048828125,
      "loss": 0.145,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9229917526245117,
      "rewards/margins": 3.3273627758026123,
      "rewards/rejected": -4.250354290008545,
      "step": 3430
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.5135135135135135e-07,
      "logits/chosen": -2.5901942253112793,
      "logits/rejected": -2.7249298095703125,
      "logps/chosen": -224.41748046875,
      "logps/rejected": -315.70458984375,
      "loss": 0.1881,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9482113122940063,
      "rewards/margins": 3.2353057861328125,
      "rewards/rejected": -4.183516502380371,
      "step": 3440
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.5017626321974148e-07,
      "logits/chosen": -2.5186502933502197,
      "logits/rejected": -2.6493256092071533,
      "logps/chosen": -283.7633056640625,
      "logps/rejected": -337.82073974609375,
      "loss": 0.1904,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0490740537643433,
      "rewards/margins": 3.2278411388397217,
      "rewards/rejected": -4.276915550231934,
      "step": 3450
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.4900117508813158e-07,
      "logits/chosen": -2.491114854812622,
      "logits/rejected": -2.58499813079834,
      "logps/chosen": -280.1947326660156,
      "logps/rejected": -321.2454833984375,
      "loss": 0.1375,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0607309341430664,
      "rewards/margins": 3.155475378036499,
      "rewards/rejected": -4.216206073760986,
      "step": 3460
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.4782608695652173e-07,
      "logits/chosen": -2.506153106689453,
      "logits/rejected": -2.590386390686035,
      "logps/chosen": -288.2539367675781,
      "logps/rejected": -342.2157287597656,
      "loss": 0.1706,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0391826629638672,
      "rewards/margins": 3.0518252849578857,
      "rewards/rejected": -4.091008186340332,
      "step": 3470
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.4665099882491186e-07,
      "logits/chosen": -2.482970714569092,
      "logits/rejected": -2.6167941093444824,
      "logps/chosen": -222.64047241210938,
      "logps/rejected": -325.3092346191406,
      "loss": 0.1822,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8933473825454712,
      "rewards/margins": 3.2153611183166504,
      "rewards/rejected": -4.108708381652832,
      "step": 3480
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.45475910693302e-07,
      "logits/chosen": -2.551558017730713,
      "logits/rejected": -2.669564723968506,
      "logps/chosen": -235.7182159423828,
      "logps/rejected": -320.32049560546875,
      "loss": 0.1613,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8789691925048828,
      "rewards/margins": 3.3068783283233643,
      "rewards/rejected": -4.185847282409668,
      "step": 3490
    },
    {
      "epoch": 2.22,
      "learning_rate": 1.4430082256169212e-07,
      "logits/chosen": -2.5294244289398193,
      "logits/rejected": -2.6788675785064697,
      "logps/chosen": -315.461669921875,
      "logps/rejected": -388.76806640625,
      "loss": 0.1664,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9383514523506165,
      "rewards/margins": 3.4844799041748047,
      "rewards/rejected": -4.4228315353393555,
      "step": 3500
    },
    {
      "epoch": 2.22,
      "eval_logits/chosen": -2.6269114017486572,
      "eval_logits/rejected": -2.752399206161499,
      "eval_logps/chosen": -280.1592102050781,
      "eval_logps/rejected": -327.4239196777344,
      "eval_loss": 0.17902666330337524,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9246384501457214,
      "eval_rewards/margins": 3.1469767093658447,
      "eval_rewards/rejected": -4.071614742279053,
      "eval_runtime": 106.2569,
      "eval_samples_per_second": 2.428,
      "eval_steps_per_second": 0.612,
      "step": 3500
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4312573443008224e-07,
      "logits/chosen": -2.5485260486602783,
      "logits/rejected": -2.686608076095581,
      "logps/chosen": -263.57916259765625,
      "logps/rejected": -319.72674560546875,
      "loss": 0.1456,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9329320192337036,
      "rewards/margins": 3.2465107440948486,
      "rewards/rejected": -4.179442882537842,
      "step": 3510
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4195064629847237e-07,
      "logits/chosen": -2.5850958824157715,
      "logits/rejected": -2.71040415763855,
      "logps/chosen": -270.2200012207031,
      "logps/rejected": -343.9569396972656,
      "loss": 0.1493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9209542274475098,
      "rewards/margins": 3.471010684967041,
      "rewards/rejected": -4.391964435577393,
      "step": 3520
    },
    {
      "epoch": 2.24,
      "learning_rate": 1.407755581668625e-07,
      "logits/chosen": -2.598151683807373,
      "logits/rejected": -2.6103134155273438,
      "logps/chosen": -352.1795349121094,
      "logps/rejected": -350.29534912109375,
      "loss": 0.1764,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1048429012298584,
      "rewards/margins": 3.219926118850708,
      "rewards/rejected": -4.324769020080566,
      "step": 3530
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3960047003525263e-07,
      "logits/chosen": -2.5510733127593994,
      "logits/rejected": -2.691777467727661,
      "logps/chosen": -255.44384765625,
      "logps/rejected": -326.2204284667969,
      "loss": 0.1565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0204987525939941,
      "rewards/margins": 3.180412530899048,
      "rewards/rejected": -4.200911045074463,
      "step": 3540
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3842538190364278e-07,
      "logits/chosen": -2.550557851791382,
      "logits/rejected": -2.7066280841827393,
      "logps/chosen": -248.0840301513672,
      "logps/rejected": -334.6856994628906,
      "loss": 0.1717,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9193902015686035,
      "rewards/margins": 3.2888922691345215,
      "rewards/rejected": -4.208282470703125,
      "step": 3550
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.372502937720329e-07,
      "logits/chosen": -2.522473096847534,
      "logits/rejected": -2.6209702491760254,
      "logps/chosen": -267.84881591796875,
      "logps/rejected": -317.5884704589844,
      "loss": 0.1948,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1308584213256836,
      "rewards/margins": 2.938067674636841,
      "rewards/rejected": -4.0689263343811035,
      "step": 3560
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.36075205640423e-07,
      "logits/chosen": -2.478235960006714,
      "logits/rejected": -2.6274712085723877,
      "logps/chosen": -241.786376953125,
      "logps/rejected": -350.6224670410156,
      "loss": 0.1565,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8250061869621277,
      "rewards/margins": 3.480761766433716,
      "rewards/rejected": -4.305768013000488,
      "step": 3570
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.3490011750881317e-07,
      "logits/chosen": -2.447965145111084,
      "logits/rejected": -2.6116302013397217,
      "logps/chosen": -261.8627014160156,
      "logps/rejected": -357.03826904296875,
      "loss": 0.1688,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9683946371078491,
      "rewards/margins": 3.220907688140869,
      "rewards/rejected": -4.189302444458008,
      "step": 3580
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.337250293772033e-07,
      "logits/chosen": -2.492553472518921,
      "logits/rejected": -2.652484178543091,
      "logps/chosen": -238.107421875,
      "logps/rejected": -336.3323059082031,
      "loss": 0.1456,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8908166885375977,
      "rewards/margins": 3.7755978107452393,
      "rewards/rejected": -4.666414737701416,
      "step": 3590
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.325499412455934e-07,
      "logits/chosen": -2.63576078414917,
      "logits/rejected": -2.697021722793579,
      "logps/chosen": -310.36767578125,
      "logps/rejected": -341.4948425292969,
      "loss": 0.2085,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9383770227432251,
      "rewards/margins": 2.758697509765625,
      "rewards/rejected": -3.6970748901367188,
      "step": 3600
    },
    {
      "epoch": 2.28,
      "eval_logits/chosen": -2.6279244422912598,
      "eval_logits/rejected": -2.753206968307495,
      "eval_logps/chosen": -280.21356201171875,
      "eval_logps/rejected": -327.5426330566406,
      "eval_loss": 0.17874082922935486,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.930072009563446,
      "eval_rewards/margins": 3.1534156799316406,
      "eval_rewards/rejected": -4.083487510681152,
      "eval_runtime": 105.5489,
      "eval_samples_per_second": 2.444,
      "eval_steps_per_second": 0.616,
      "step": 3600
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.3137485311398355e-07,
      "logits/chosen": -2.5338902473449707,
      "logits/rejected": -2.6711642742156982,
      "logps/chosen": -276.4370422363281,
      "logps/rejected": -337.9762268066406,
      "loss": 0.1853,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.921554684638977,
      "rewards/margins": 2.90724515914917,
      "rewards/rejected": -3.8287997245788574,
      "step": 3610
    },
    {
      "epoch": 2.3,
      "learning_rate": 1.3019976498237368e-07,
      "logits/chosen": -2.5164458751678467,
      "logits/rejected": -2.641270160675049,
      "logps/chosen": -263.2311706542969,
      "logps/rejected": -348.1232604980469,
      "loss": 0.1458,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0515857934951782,
      "rewards/margins": 3.4831089973449707,
      "rewards/rejected": -4.534694194793701,
      "step": 3620
    },
    {
      "epoch": 2.3,
      "learning_rate": 1.290246768507638e-07,
      "logits/chosen": -2.494929790496826,
      "logits/rejected": -2.6594760417938232,
      "logps/chosen": -239.16903686523438,
      "logps/rejected": -327.6562194824219,
      "loss": 0.1539,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7176671028137207,
      "rewards/margins": 3.316002607345581,
      "rewards/rejected": -4.033669471740723,
      "step": 3630
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2784958871915393e-07,
      "logits/chosen": -2.504030704498291,
      "logits/rejected": -2.6586759090423584,
      "logps/chosen": -201.3271942138672,
      "logps/rejected": -290.07781982421875,
      "loss": 0.1624,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7369797825813293,
      "rewards/margins": 3.0305566787719727,
      "rewards/rejected": -3.7675366401672363,
      "step": 3640
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.2667450058754406e-07,
      "logits/chosen": -2.518988609313965,
      "logits/rejected": -2.6703150272369385,
      "logps/chosen": -286.662109375,
      "logps/rejected": -380.80230712890625,
      "loss": 0.1786,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9173741340637207,
      "rewards/margins": 3.2654221057891846,
      "rewards/rejected": -4.182796478271484,
      "step": 3650
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.254994124559342e-07,
      "logits/chosen": -2.5328404903411865,
      "logits/rejected": -2.673396110534668,
      "logps/chosen": -247.16885375976562,
      "logps/rejected": -331.464111328125,
      "loss": 0.1608,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7315680980682373,
      "rewards/margins": 3.148334503173828,
      "rewards/rejected": -3.8799026012420654,
      "step": 3660
    },
    {
      "epoch": 2.33,
      "learning_rate": 1.2432432432432432e-07,
      "logits/chosen": -2.5625972747802734,
      "logits/rejected": -2.658459186553955,
      "logps/chosen": -292.9676818847656,
      "logps/rejected": -352.4302673339844,
      "loss": 0.1675,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0281872749328613,
      "rewards/margins": 3.2200229167938232,
      "rewards/rejected": -4.2482099533081055,
      "step": 3670
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.2314923619271444e-07,
      "logits/chosen": -2.5082576274871826,
      "logits/rejected": -2.6304171085357666,
      "logps/chosen": -286.1788024902344,
      "logps/rejected": -336.09027099609375,
      "loss": 0.1523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9823424220085144,
      "rewards/margins": 3.4198760986328125,
      "rewards/rejected": -4.402218818664551,
      "step": 3680
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.2197414806110457e-07,
      "logits/chosen": -2.5131266117095947,
      "logits/rejected": -2.6457695960998535,
      "logps/chosen": -305.1301574707031,
      "logps/rejected": -357.3551940917969,
      "loss": 0.1316,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0326460599899292,
      "rewards/margins": 3.8570199012756348,
      "rewards/rejected": -4.889666557312012,
      "step": 3690
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.207990599294947e-07,
      "logits/chosen": -2.578427791595459,
      "logits/rejected": -2.716935634613037,
      "logps/chosen": -257.48114013671875,
      "logps/rejected": -343.4159240722656,
      "loss": 0.1565,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8567155003547668,
      "rewards/margins": 3.4758243560791016,
      "rewards/rejected": -4.332540035247803,
      "step": 3700
    },
    {
      "epoch": 2.35,
      "eval_logits/chosen": -2.626546621322632,
      "eval_logits/rejected": -2.7520954608917236,
      "eval_logps/chosen": -280.2137451171875,
      "eval_logps/rejected": -327.6163635253906,
      "eval_loss": 0.17821472883224487,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9300913214683533,
      "eval_rewards/margins": 3.160770893096924,
      "eval_rewards/rejected": -4.090862274169922,
      "eval_runtime": 112.8194,
      "eval_samples_per_second": 2.287,
      "eval_steps_per_second": 0.576,
      "step": 3700
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1962397179788483e-07,
      "logits/chosen": -2.5717310905456543,
      "logits/rejected": -2.68367600440979,
      "logps/chosen": -280.4960632324219,
      "logps/rejected": -335.37567138671875,
      "loss": 0.1697,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9972933530807495,
      "rewards/margins": 3.4042770862579346,
      "rewards/rejected": -4.401570796966553,
      "step": 3710
    },
    {
      "epoch": 2.36,
      "learning_rate": 1.1844888366627497e-07,
      "logits/chosen": -2.5783801078796387,
      "logits/rejected": -2.716322422027588,
      "logps/chosen": -301.6819763183594,
      "logps/rejected": -349.97430419921875,
      "loss": 0.1525,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0113129615783691,
      "rewards/margins": 3.409414768218994,
      "rewards/rejected": -4.420727729797363,
      "step": 3720
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.172737955346651e-07,
      "logits/chosen": -2.522198438644409,
      "logits/rejected": -2.6234254837036133,
      "logps/chosen": -292.25152587890625,
      "logps/rejected": -342.09918212890625,
      "loss": 0.1796,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0002429485321045,
      "rewards/margins": 3.331533908843994,
      "rewards/rejected": -4.3317766189575195,
      "step": 3730
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.1609870740305522e-07,
      "logits/chosen": -2.5087151527404785,
      "logits/rejected": -2.605372190475464,
      "logps/chosen": -318.2735290527344,
      "logps/rejected": -348.9207763671875,
      "loss": 0.122,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.026114583015442,
      "rewards/margins": 3.6023952960968018,
      "rewards/rejected": -4.628509044647217,
      "step": 3740
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1492361927144535e-07,
      "logits/chosen": -2.5257058143615723,
      "logits/rejected": -2.6447081565856934,
      "logps/chosen": -265.0050048828125,
      "logps/rejected": -334.9100341796875,
      "loss": 0.1479,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9693697094917297,
      "rewards/margins": 3.398129940032959,
      "rewards/rejected": -4.367499351501465,
      "step": 3750
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1374853113983548e-07,
      "logits/chosen": -2.692702531814575,
      "logits/rejected": -2.74967622756958,
      "logps/chosen": -323.7606201171875,
      "logps/rejected": -328.6816101074219,
      "loss": 0.1972,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.9621502757072449,
      "rewards/margins": 2.913362503051758,
      "rewards/rejected": -3.8755123615264893,
      "step": 3760
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1257344300822562e-07,
      "logits/chosen": -2.519091844558716,
      "logits/rejected": -2.6225621700286865,
      "logps/chosen": -229.6961669921875,
      "logps/rejected": -285.30828857421875,
      "loss": 0.1986,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7775883674621582,
      "rewards/margins": 3.202441453933716,
      "rewards/rejected": -3.980029582977295,
      "step": 3770
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.1139835487661573e-07,
      "logits/chosen": -2.4712424278259277,
      "logits/rejected": -2.6218645572662354,
      "logps/chosen": -263.41693115234375,
      "logps/rejected": -342.00360107421875,
      "loss": 0.152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8185707926750183,
      "rewards/margins": 3.5038063526153564,
      "rewards/rejected": -4.322376728057861,
      "step": 3780
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.1022326674500588e-07,
      "logits/chosen": -2.426617383956909,
      "logits/rejected": -2.5692572593688965,
      "logps/chosen": -295.9425354003906,
      "logps/rejected": -353.3335266113281,
      "loss": 0.1629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0660467147827148,
      "rewards/margins": 3.3372890949249268,
      "rewards/rejected": -4.403336048126221,
      "step": 3790
    },
    {
      "epoch": 2.41,
      "learning_rate": 1.09048178613396e-07,
      "logits/chosen": -2.503966808319092,
      "logits/rejected": -2.6766552925109863,
      "logps/chosen": -229.75216674804688,
      "logps/rejected": -332.28546142578125,
      "loss": 0.153,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9133013486862183,
      "rewards/margins": 3.446155071258545,
      "rewards/rejected": -4.3594560623168945,
      "step": 3800
    },
    {
      "epoch": 2.41,
      "eval_logits/chosen": -2.6268134117126465,
      "eval_logits/rejected": -2.752187728881836,
      "eval_logps/chosen": -280.1937255859375,
      "eval_logps/rejected": -327.655029296875,
      "eval_loss": 0.1777840107679367,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9280871152877808,
      "eval_rewards/margins": 3.1666419506073,
      "eval_rewards/rejected": -4.094728946685791,
      "eval_runtime": 112.5361,
      "eval_samples_per_second": 2.293,
      "eval_steps_per_second": 0.578,
      "step": 3800
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.0787309048178613e-07,
      "logits/chosen": -2.5616233348846436,
      "logits/rejected": -2.6658310890197754,
      "logps/chosen": -285.86749267578125,
      "logps/rejected": -357.8493957519531,
      "loss": 0.1751,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9552777409553528,
      "rewards/margins": 3.400822401046753,
      "rewards/rejected": -4.356100082397461,
      "step": 3810
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.0669800235017626e-07,
      "logits/chosen": -2.5523791313171387,
      "logits/rejected": -2.677839517593384,
      "logps/chosen": -275.10882568359375,
      "logps/rejected": -344.29669189453125,
      "loss": 0.1694,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9939683675765991,
      "rewards/margins": 3.329462766647339,
      "rewards/rejected": -4.323431491851807,
      "step": 3820
    },
    {
      "epoch": 2.43,
      "learning_rate": 1.0552291421856639e-07,
      "logits/chosen": -2.5229134559631348,
      "logits/rejected": -2.6731162071228027,
      "logps/chosen": -225.63693237304688,
      "logps/rejected": -319.5437927246094,
      "loss": 0.1711,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.990398108959198,
      "rewards/margins": 3.144343614578247,
      "rewards/rejected": -4.13474178314209,
      "step": 3830
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0434782608695651e-07,
      "logits/chosen": -2.452129364013672,
      "logits/rejected": -2.5820062160491943,
      "logps/chosen": -257.4875793457031,
      "logps/rejected": -332.9882507324219,
      "loss": 0.1529,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1371349096298218,
      "rewards/margins": 3.2712242603302,
      "rewards/rejected": -4.408358573913574,
      "step": 3840
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0317273795534664e-07,
      "logits/chosen": -2.532578229904175,
      "logits/rejected": -2.664321184158325,
      "logps/chosen": -301.3134765625,
      "logps/rejected": -369.0633239746094,
      "loss": 0.1588,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8863778114318848,
      "rewards/margins": 3.329010009765625,
      "rewards/rejected": -4.21538782119751,
      "step": 3850
    },
    {
      "epoch": 2.45,
      "learning_rate": 1.0199764982373678e-07,
      "logits/chosen": -2.521547794342041,
      "logits/rejected": -2.6621651649475098,
      "logps/chosen": -242.22805786132812,
      "logps/rejected": -335.8890075683594,
      "loss": 0.1685,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9544798731803894,
      "rewards/margins": 3.512755870819092,
      "rewards/rejected": -4.467236042022705,
      "step": 3860
    },
    {
      "epoch": 2.46,
      "learning_rate": 1.0082256169212691e-07,
      "logits/chosen": -2.4783453941345215,
      "logits/rejected": -2.5657029151916504,
      "logps/chosen": -323.0866394042969,
      "logps/rejected": -365.36627197265625,
      "loss": 0.1383,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9324504733085632,
      "rewards/margins": 3.3638014793395996,
      "rewards/rejected": -4.296252250671387,
      "step": 3870
    },
    {
      "epoch": 2.46,
      "learning_rate": 9.964747356051703e-08,
      "logits/chosen": -2.4510788917541504,
      "logits/rejected": -2.666691541671753,
      "logps/chosen": -239.125244140625,
      "logps/rejected": -347.33795166015625,
      "loss": 0.1261,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8064861297607422,
      "rewards/margins": 3.6986262798309326,
      "rewards/rejected": -4.505112648010254,
      "step": 3880
    },
    {
      "epoch": 2.47,
      "learning_rate": 9.847238542890717e-08,
      "logits/chosen": -2.5597939491271973,
      "logits/rejected": -2.6242892742156982,
      "logps/chosen": -333.1199645996094,
      "logps/rejected": -330.65899658203125,
      "loss": 0.1629,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0350576639175415,
      "rewards/margins": 2.98962140083313,
      "rewards/rejected": -4.024679183959961,
      "step": 3890
    },
    {
      "epoch": 2.47,
      "learning_rate": 9.72972972972973e-08,
      "logits/chosen": -2.49501633644104,
      "logits/rejected": -2.630890369415283,
      "logps/chosen": -239.18350219726562,
      "logps/rejected": -314.84722900390625,
      "loss": 0.1787,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8169991374015808,
      "rewards/margins": 3.1660754680633545,
      "rewards/rejected": -3.983074903488159,
      "step": 3900
    },
    {
      "epoch": 2.47,
      "eval_logits/chosen": -2.6265957355499268,
      "eval_logits/rejected": -2.75201153755188,
      "eval_logps/chosen": -280.2315979003906,
      "eval_logps/rejected": -327.6258850097656,
      "eval_loss": 0.17827431857585907,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9318748712539673,
      "eval_rewards/margins": 3.1599371433258057,
      "eval_rewards/rejected": -4.0918121337890625,
      "eval_runtime": 112.6173,
      "eval_samples_per_second": 2.291,
      "eval_steps_per_second": 0.577,
      "step": 3900
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.612220916568742e-08,
      "logits/chosen": -2.465816020965576,
      "logits/rejected": -2.6596102714538574,
      "logps/chosen": -228.0482940673828,
      "logps/rejected": -357.10369873046875,
      "loss": 0.1275,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9100969433784485,
      "rewards/margins": 3.647930145263672,
      "rewards/rejected": -4.5580267906188965,
      "step": 3910
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.494712103407755e-08,
      "logits/chosen": -2.548189640045166,
      "logits/rejected": -2.6721997261047363,
      "logps/chosen": -297.14556884765625,
      "logps/rejected": -355.9110412597656,
      "loss": 0.1681,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8179681897163391,
      "rewards/margins": 3.248883008956909,
      "rewards/rejected": -4.0668511390686035,
      "step": 3920
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.377203290246769e-08,
      "logits/chosen": -2.6154046058654785,
      "logits/rejected": -2.7137889862060547,
      "logps/chosen": -253.2783203125,
      "logps/rejected": -314.24371337890625,
      "loss": 0.1561,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0781346559524536,
      "rewards/margins": 3.3648693561553955,
      "rewards/rejected": -4.4430036544799805,
      "step": 3930
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.25969447708578e-08,
      "logits/chosen": -2.544506549835205,
      "logits/rejected": -2.7005770206451416,
      "logps/chosen": -217.7257537841797,
      "logps/rejected": -310.5904235839844,
      "loss": 0.1625,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.0001634359359741,
      "rewards/margins": 3.2627601623535156,
      "rewards/rejected": -4.262923717498779,
      "step": 3940
    },
    {
      "epoch": 2.51,
      "learning_rate": 9.142185663924793e-08,
      "logits/chosen": -2.499025583267212,
      "logits/rejected": -2.5754616260528564,
      "logps/chosen": -281.67071533203125,
      "logps/rejected": -330.0785827636719,
      "loss": 0.1736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8778359293937683,
      "rewards/margins": 3.1316535472869873,
      "rewards/rejected": -4.0094895362854,
      "step": 3950
    },
    {
      "epoch": 2.51,
      "learning_rate": 9.024676850763807e-08,
      "logits/chosen": -2.489858627319336,
      "logits/rejected": -2.6101675033569336,
      "logps/chosen": -250.6933135986328,
      "logps/rejected": -336.415771484375,
      "loss": 0.1588,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9261037111282349,
      "rewards/margins": 3.3117244243621826,
      "rewards/rejected": -4.237828254699707,
      "step": 3960
    },
    {
      "epoch": 2.52,
      "learning_rate": 8.90716803760282e-08,
      "logits/chosen": -2.568180561065674,
      "logits/rejected": -2.672750949859619,
      "logps/chosen": -262.4926452636719,
      "logps/rejected": -341.45391845703125,
      "loss": 0.1849,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1056115627288818,
      "rewards/margins": 3.0754377841949463,
      "rewards/rejected": -4.18104887008667,
      "step": 3970
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.789659224441833e-08,
      "logits/chosen": -2.67130446434021,
      "logits/rejected": -2.81573748588562,
      "logps/chosen": -257.25042724609375,
      "logps/rejected": -339.4892883300781,
      "loss": 0.1907,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8284015655517578,
      "rewards/margins": 3.420711040496826,
      "rewards/rejected": -4.249113082885742,
      "step": 3980
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.672150411280846e-08,
      "logits/chosen": -2.4839107990264893,
      "logits/rejected": -2.616621255874634,
      "logps/chosen": -243.0556640625,
      "logps/rejected": -328.2231750488281,
      "loss": 0.1546,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9281770586967468,
      "rewards/margins": 3.184389114379883,
      "rewards/rejected": -4.1125664710998535,
      "step": 3990
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.554641598119859e-08,
      "logits/chosen": -2.7165770530700684,
      "logits/rejected": -2.82002329826355,
      "logps/chosen": -283.5413818359375,
      "logps/rejected": -335.9131774902344,
      "loss": 0.172,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0186265707015991,
      "rewards/margins": 3.2503600120544434,
      "rewards/rejected": -4.268986701965332,
      "step": 4000
    },
    {
      "epoch": 2.54,
      "eval_logits/chosen": -2.62734055519104,
      "eval_logits/rejected": -2.7525932788848877,
      "eval_logps/chosen": -280.25054931640625,
      "eval_logps/rejected": -327.7428894042969,
      "eval_loss": 0.1780453771352768,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9337711930274963,
      "eval_rewards/margins": 3.169741153717041,
      "eval_rewards/rejected": -4.103512763977051,
      "eval_runtime": 106.8587,
      "eval_samples_per_second": 2.414,
      "eval_steps_per_second": 0.608,
      "step": 4000
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.437132784958871e-08,
      "logits/chosen": -2.534666061401367,
      "logits/rejected": -2.65165638923645,
      "logps/chosen": -278.1713562011719,
      "logps/rejected": -323.4463195800781,
      "loss": 0.1595,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7092602252960205,
      "rewards/margins": 3.1355316638946533,
      "rewards/rejected": -3.8447914123535156,
      "step": 4010
    },
    {
      "epoch": 2.55,
      "learning_rate": 8.319623971797884e-08,
      "logits/chosen": -2.4885506629943848,
      "logits/rejected": -2.5927164554595947,
      "logps/chosen": -268.8062438964844,
      "logps/rejected": -318.78631591796875,
      "loss": 0.1477,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7434967756271362,
      "rewards/margins": 3.492476224899292,
      "rewards/rejected": -4.2359724044799805,
      "step": 4020
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.202115158636898e-08,
      "logits/chosen": -2.511106014251709,
      "logits/rejected": -2.6457321643829346,
      "logps/chosen": -257.6388854980469,
      "logps/rejected": -332.87176513671875,
      "loss": 0.1637,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9297393560409546,
      "rewards/margins": 3.4277915954589844,
      "rewards/rejected": -4.357531547546387,
      "step": 4030
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.08460634547591e-08,
      "logits/chosen": -2.5118274688720703,
      "logits/rejected": -2.6484532356262207,
      "logps/chosen": -223.2838592529297,
      "logps/rejected": -297.0276184082031,
      "loss": 0.2101,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8684900999069214,
      "rewards/margins": 2.9051215648651123,
      "rewards/rejected": -3.7736117839813232,
      "step": 4040
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.967097532314922e-08,
      "logits/chosen": -2.492349147796631,
      "logits/rejected": -2.5758488178253174,
      "logps/chosen": -300.62451171875,
      "logps/rejected": -323.60272216796875,
      "loss": 0.1534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0182479619979858,
      "rewards/margins": 3.225146532058716,
      "rewards/rejected": -4.243393898010254,
      "step": 4050
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.849588719153937e-08,
      "logits/chosen": -2.4840006828308105,
      "logits/rejected": -2.6301465034484863,
      "logps/chosen": -271.11810302734375,
      "logps/rejected": -355.0530090332031,
      "loss": 0.1227,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0057458877563477,
      "rewards/margins": 3.6259236335754395,
      "rewards/rejected": -4.631669521331787,
      "step": 4060
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.73207990599295e-08,
      "logits/chosen": -2.5467135906219482,
      "logits/rejected": -2.7198383808135986,
      "logps/chosen": -270.92913818359375,
      "logps/rejected": -368.2033386230469,
      "loss": 0.1661,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9587764739990234,
      "rewards/margins": 3.2931313514709473,
      "rewards/rejected": -4.251907825469971,
      "step": 4070
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.614571092831962e-08,
      "logits/chosen": -2.6016898155212402,
      "logits/rejected": -2.694291114807129,
      "logps/chosen": -300.1907043457031,
      "logps/rejected": -321.87835693359375,
      "loss": 0.186,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.9011460542678833,
      "rewards/margins": 2.8087880611419678,
      "rewards/rejected": -3.7099337577819824,
      "step": 4080
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.497062279670975e-08,
      "logits/chosen": -2.558283805847168,
      "logits/rejected": -2.6539347171783447,
      "logps/chosen": -282.11505126953125,
      "logps/rejected": -332.7256774902344,
      "loss": 0.1441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7828934788703918,
      "rewards/margins": 3.321030855178833,
      "rewards/rejected": -4.103924751281738,
      "step": 4090
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.379553466509989e-08,
      "logits/chosen": -2.599134683609009,
      "logits/rejected": -2.700207471847534,
      "logps/chosen": -271.44134521484375,
      "logps/rejected": -322.71435546875,
      "loss": 0.2643,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8485885858535767,
      "rewards/margins": 2.7676377296447754,
      "rewards/rejected": -3.6162261962890625,
      "step": 4100
    },
    {
      "epoch": 2.6,
      "eval_logits/chosen": -2.6267471313476562,
      "eval_logits/rejected": -2.752119779586792,
      "eval_logps/chosen": -280.14215087890625,
      "eval_logps/rejected": -327.6763916015625,
      "eval_loss": 0.17712660133838654,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9229325652122498,
      "eval_rewards/margins": 3.173929452896118,
      "eval_rewards/rejected": -4.096861839294434,
      "eval_runtime": 112.2306,
      "eval_samples_per_second": 2.299,
      "eval_steps_per_second": 0.579,
      "step": 4100
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.262044653349e-08,
      "logits/chosen": -2.508070468902588,
      "logits/rejected": -2.6169333457946777,
      "logps/chosen": -264.45440673828125,
      "logps/rejected": -319.60101318359375,
      "loss": 0.1644,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8941587209701538,
      "rewards/margins": 3.1691031455993652,
      "rewards/rejected": -4.063261985778809,
      "step": 4110
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.144535840188013e-08,
      "logits/chosen": -2.4586760997772217,
      "logits/rejected": -2.591771364212036,
      "logps/chosen": -260.462890625,
      "logps/rejected": -318.4164733886719,
      "loss": 0.1614,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.945212185382843,
      "rewards/margins": 3.257131576538086,
      "rewards/rejected": -4.202343940734863,
      "step": 4120
    },
    {
      "epoch": 2.62,
      "learning_rate": 7.027027027027027e-08,
      "logits/chosen": -2.5315022468566895,
      "logits/rejected": -2.6705734729766846,
      "logps/chosen": -276.8602600097656,
      "logps/rejected": -347.9608154296875,
      "loss": 0.1788,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8511897921562195,
      "rewards/margins": 3.3756279945373535,
      "rewards/rejected": -4.2268171310424805,
      "step": 4130
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.909518213866039e-08,
      "logits/chosen": -2.595262050628662,
      "logits/rejected": -2.738239049911499,
      "logps/chosen": -256.0868835449219,
      "logps/rejected": -334.80352783203125,
      "loss": 0.1556,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9345976710319519,
      "rewards/margins": 3.431105852127075,
      "rewards/rejected": -4.365703105926514,
      "step": 4140
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.792009400705053e-08,
      "logits/chosen": -2.5195977687835693,
      "logits/rejected": -2.6573235988616943,
      "logps/chosen": -236.57077026367188,
      "logps/rejected": -317.4797668457031,
      "loss": 0.134,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8615215420722961,
      "rewards/margins": 3.5458827018737793,
      "rewards/rejected": -4.407403945922852,
      "step": 4150
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.674500587544066e-08,
      "logits/chosen": -2.561389923095703,
      "logits/rejected": -2.680393934249878,
      "logps/chosen": -269.3076477050781,
      "logps/rejected": -341.43267822265625,
      "loss": 0.1609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9654695391654968,
      "rewards/margins": 3.216449022293091,
      "rewards/rejected": -4.181918144226074,
      "step": 4160
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.556991774383078e-08,
      "logits/chosen": -2.5944087505340576,
      "logits/rejected": -2.710848331451416,
      "logps/chosen": -254.6079864501953,
      "logps/rejected": -310.8287048339844,
      "loss": 0.1276,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.0094635486602783,
      "rewards/margins": 3.4649760723114014,
      "rewards/rejected": -4.474440097808838,
      "step": 4170
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.439482961222091e-08,
      "logits/chosen": -2.527759552001953,
      "logits/rejected": -2.6481010913848877,
      "logps/chosen": -273.606689453125,
      "logps/rejected": -315.1614074707031,
      "loss": 0.1367,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.0404853820800781,
      "rewards/margins": 3.320519208908081,
      "rewards/rejected": -4.361004829406738,
      "step": 4180
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.321974148061104e-08,
      "logits/chosen": -2.5424933433532715,
      "logits/rejected": -2.681966543197632,
      "logps/chosen": -265.18157958984375,
      "logps/rejected": -328.3844299316406,
      "loss": 0.1431,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9357797503471375,
      "rewards/margins": 3.3465702533721924,
      "rewards/rejected": -4.282350063323975,
      "step": 4190
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.204465334900117e-08,
      "logits/chosen": -2.5607879161834717,
      "logits/rejected": -2.680274724960327,
      "logps/chosen": -282.68072509765625,
      "logps/rejected": -337.3699645996094,
      "loss": 0.1619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9207731485366821,
      "rewards/margins": 3.2776737213134766,
      "rewards/rejected": -4.198446750640869,
      "step": 4200
    },
    {
      "epoch": 2.66,
      "eval_logits/chosen": -2.6269819736480713,
      "eval_logits/rejected": -2.75227952003479,
      "eval_logps/chosen": -280.2390441894531,
      "eval_logps/rejected": -327.7908630371094,
      "eval_loss": 0.1775965392589569,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9326200485229492,
      "eval_rewards/margins": 3.1756935119628906,
      "eval_rewards/rejected": -4.10831356048584,
      "eval_runtime": 107.3879,
      "eval_samples_per_second": 2.403,
      "eval_steps_per_second": 0.605,
      "step": 4200
    },
    {
      "epoch": 2.67,
      "learning_rate": 6.086956521739131e-08,
      "logits/chosen": -2.521428108215332,
      "logits/rejected": -2.6493849754333496,
      "logps/chosen": -271.08868408203125,
      "logps/rejected": -334.4770812988281,
      "loss": 0.1582,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9069653749465942,
      "rewards/margins": 3.438695192337036,
      "rewards/rejected": -4.345660209655762,
      "step": 4210
    },
    {
      "epoch": 2.68,
      "learning_rate": 5.969447708578144e-08,
      "logits/chosen": -2.5177974700927734,
      "logits/rejected": -2.6404643058776855,
      "logps/chosen": -238.5574951171875,
      "logps/rejected": -324.5984802246094,
      "loss": 0.1771,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8974288702011108,
      "rewards/margins": 3.215510845184326,
      "rewards/rejected": -4.112939834594727,
      "step": 4220
    },
    {
      "epoch": 2.68,
      "learning_rate": 5.851938895417156e-08,
      "logits/chosen": -2.469808578491211,
      "logits/rejected": -2.578573226928711,
      "logps/chosen": -275.1860656738281,
      "logps/rejected": -333.9859619140625,
      "loss": 0.1926,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8939563632011414,
      "rewards/margins": 3.11181378364563,
      "rewards/rejected": -4.005770206451416,
      "step": 4230
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.734430082256169e-08,
      "logits/chosen": -2.484970808029175,
      "logits/rejected": -2.689824104309082,
      "logps/chosen": -208.4597930908203,
      "logps/rejected": -323.81329345703125,
      "loss": 0.1206,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9648736715316772,
      "rewards/margins": 3.6686108112335205,
      "rewards/rejected": -4.633484840393066,
      "step": 4240
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.616921269095182e-08,
      "logits/chosen": -2.521632671356201,
      "logits/rejected": -2.658073902130127,
      "logps/chosen": -312.0960998535156,
      "logps/rejected": -367.4841003417969,
      "loss": 0.1595,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0449492931365967,
      "rewards/margins": 3.408780574798584,
      "rewards/rejected": -4.45373010635376,
      "step": 4250
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.499412455934195e-08,
      "logits/chosen": -2.5796234607696533,
      "logits/rejected": -2.729409694671631,
      "logps/chosen": -274.0209655761719,
      "logps/rejected": -371.04595947265625,
      "loss": 0.1566,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8667489290237427,
      "rewards/margins": 3.560990810394287,
      "rewards/rejected": -4.427739143371582,
      "step": 4260
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.3819036427732076e-08,
      "logits/chosen": -2.5243144035339355,
      "logits/rejected": -2.70009183883667,
      "logps/chosen": -271.05352783203125,
      "logps/rejected": -354.3619079589844,
      "loss": 0.0839,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.0780527591705322,
      "rewards/margins": 3.8228249549865723,
      "rewards/rejected": -4.900877952575684,
      "step": 4270
    },
    {
      "epoch": 2.72,
      "learning_rate": 5.2643948296122204e-08,
      "logits/chosen": -2.457848072052002,
      "logits/rejected": -2.5908544063568115,
      "logps/chosen": -303.19805908203125,
      "logps/rejected": -347.88580322265625,
      "loss": 0.1288,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.088138461112976,
      "rewards/margins": 3.5334324836730957,
      "rewards/rejected": -4.621571063995361,
      "step": 4280
    },
    {
      "epoch": 2.72,
      "learning_rate": 5.146886016451234e-08,
      "logits/chosen": -2.5475432872772217,
      "logits/rejected": -2.6708507537841797,
      "logps/chosen": -278.64874267578125,
      "logps/rejected": -350.92523193359375,
      "loss": 0.1681,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.9762653112411499,
      "rewards/margins": 3.2399990558624268,
      "rewards/rejected": -4.216264247894287,
      "step": 4290
    },
    {
      "epoch": 2.73,
      "learning_rate": 5.0293772032902466e-08,
      "logits/chosen": -2.638136386871338,
      "logits/rejected": -2.7750489711761475,
      "logps/chosen": -281.7099914550781,
      "logps/rejected": -354.05987548828125,
      "loss": 0.2413,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9155136346817017,
      "rewards/margins": 3.1325125694274902,
      "rewards/rejected": -4.048026084899902,
      "step": 4300
    },
    {
      "epoch": 2.73,
      "eval_logits/chosen": -2.6276509761810303,
      "eval_logits/rejected": -2.7528743743896484,
      "eval_logps/chosen": -280.2050476074219,
      "eval_logps/rejected": -327.73150634765625,
      "eval_loss": 0.17780528962612152,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9292186498641968,
      "eval_rewards/margins": 3.173159122467041,
      "eval_rewards/rejected": -4.102377414703369,
      "eval_runtime": 102.7924,
      "eval_samples_per_second": 2.51,
      "eval_steps_per_second": 0.632,
      "step": 4300
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.91186839012926e-08,
      "logits/chosen": -2.4912240505218506,
      "logits/rejected": -2.6212782859802246,
      "logps/chosen": -245.62734985351562,
      "logps/rejected": -307.91058349609375,
      "loss": 0.1726,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.011668086051941,
      "rewards/margins": 3.3525986671447754,
      "rewards/rejected": -4.364266395568848,
      "step": 4310
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.794359576968272e-08,
      "logits/chosen": -2.5623908042907715,
      "logits/rejected": -2.6696536540985107,
      "logps/chosen": -210.87478637695312,
      "logps/rejected": -282.48223876953125,
      "loss": 0.2415,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.8712896108627319,
      "rewards/margins": 2.706787586212158,
      "rewards/rejected": -3.5780768394470215,
      "step": 4320
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.676850763807285e-08,
      "logits/chosen": -2.4659931659698486,
      "logits/rejected": -2.5820415019989014,
      "logps/chosen": -256.8589172363281,
      "logps/rejected": -317.1506652832031,
      "loss": 0.1383,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9896674156188965,
      "rewards/margins": 3.1960551738739014,
      "rewards/rejected": -4.185723304748535,
      "step": 4330
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.5593419506462984e-08,
      "logits/chosen": -2.5236153602600098,
      "logits/rejected": -2.5750889778137207,
      "logps/chosen": -304.23028564453125,
      "logps/rejected": -321.7759094238281,
      "loss": 0.1039,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9752210378646851,
      "rewards/margins": 3.58766508102417,
      "rewards/rejected": -4.5628862380981445,
      "step": 4340
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.441833137485311e-08,
      "logits/chosen": -2.5789008140563965,
      "logits/rejected": -2.7373173236846924,
      "logps/chosen": -266.10577392578125,
      "logps/rejected": -347.2480773925781,
      "loss": 0.1426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9410840272903442,
      "rewards/margins": 3.7492923736572266,
      "rewards/rejected": -4.6903767585754395,
      "step": 4350
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.3243243243243246e-08,
      "logits/chosen": -2.4919543266296387,
      "logits/rejected": -2.5841705799102783,
      "logps/chosen": -284.065185546875,
      "logps/rejected": -320.2301330566406,
      "loss": 0.1617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9217942357063293,
      "rewards/margins": 3.196800708770752,
      "rewards/rejected": -4.118594646453857,
      "step": 4360
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.206815511163337e-08,
      "logits/chosen": -2.494184970855713,
      "logits/rejected": -2.621028184890747,
      "logps/chosen": -302.36456298828125,
      "logps/rejected": -359.72760009765625,
      "loss": 0.1554,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.92668616771698,
      "rewards/margins": 3.405775547027588,
      "rewards/rejected": -4.332461357116699,
      "step": 4370
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.0893066980023495e-08,
      "logits/chosen": -2.5035388469696045,
      "logits/rejected": -2.606834888458252,
      "logps/chosen": -252.92300415039062,
      "logps/rejected": -309.306884765625,
      "loss": 0.1767,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8464910387992859,
      "rewards/margins": 3.433950901031494,
      "rewards/rejected": -4.280441761016846,
      "step": 4380
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.971797884841363e-08,
      "logits/chosen": -2.4771904945373535,
      "logits/rejected": -2.5943856239318848,
      "logps/chosen": -239.48361206054688,
      "logps/rejected": -322.05364990234375,
      "loss": 0.1616,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.985284686088562,
      "rewards/margins": 3.391991376876831,
      "rewards/rejected": -4.3772759437561035,
      "step": 4390
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.854289071680376e-08,
      "logits/chosen": -2.5095999240875244,
      "logits/rejected": -2.6385550498962402,
      "logps/chosen": -248.973388671875,
      "logps/rejected": -310.41015625,
      "loss": 0.1187,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.838142991065979,
      "rewards/margins": 3.3973262310028076,
      "rewards/rejected": -4.235468864440918,
      "step": 4400
    },
    {
      "epoch": 2.79,
      "eval_logits/chosen": -2.626682996749878,
      "eval_logits/rejected": -2.7521493434906006,
      "eval_logps/chosen": -280.2554016113281,
      "eval_logps/rejected": -327.7757568359375,
      "eval_loss": 0.1778237372636795,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9342581629753113,
      "eval_rewards/margins": 3.172544002532959,
      "eval_rewards/rejected": -4.106801986694336,
      "eval_runtime": 112.2038,
      "eval_samples_per_second": 2.299,
      "eval_steps_per_second": 0.579,
      "step": 4400
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.736780258519389e-08,
      "logits/chosen": -2.5299601554870605,
      "logits/rejected": -2.691376209259033,
      "logps/chosen": -258.29229736328125,
      "logps/rejected": -345.01190185546875,
      "loss": 0.142,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9257384538650513,
      "rewards/margins": 3.5034165382385254,
      "rewards/rejected": -4.429154872894287,
      "step": 4410
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.619271445358402e-08,
      "logits/chosen": -2.569033145904541,
      "logits/rejected": -2.6851582527160645,
      "logps/chosen": -229.8550567626953,
      "logps/rejected": -286.8238525390625,
      "loss": 0.1679,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8025544285774231,
      "rewards/margins": 2.983921766281128,
      "rewards/rejected": -3.7864761352539062,
      "step": 4420
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.501762632197414e-08,
      "logits/chosen": -2.463283061981201,
      "logits/rejected": -2.6219160556793213,
      "logps/chosen": -233.2437286376953,
      "logps/rejected": -327.1086120605469,
      "loss": 0.1394,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8409982919692993,
      "rewards/margins": 3.315971851348877,
      "rewards/rejected": -4.156970024108887,
      "step": 4430
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.3842538190364275e-08,
      "logits/chosen": -2.4461920261383057,
      "logits/rejected": -2.5745911598205566,
      "logps/chosen": -240.30807495117188,
      "logps/rejected": -320.61962890625,
      "loss": 0.1534,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7499316930770874,
      "rewards/margins": 3.3000946044921875,
      "rewards/rejected": -4.0500264167785645,
      "step": 4440
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.26674500587544e-08,
      "logits/chosen": -2.5419421195983887,
      "logits/rejected": -2.6491737365722656,
      "logps/chosen": -238.54965209960938,
      "logps/rejected": -307.2189025878906,
      "loss": 0.1534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8124248385429382,
      "rewards/margins": 3.1456923484802246,
      "rewards/rejected": -3.9581172466278076,
      "step": 4450
    },
    {
      "epoch": 2.83,
      "learning_rate": 3.149236192714454e-08,
      "logits/chosen": -2.549132823944092,
      "logits/rejected": -2.6722254753112793,
      "logps/chosen": -311.76251220703125,
      "logps/rejected": -350.7034912109375,
      "loss": 0.1921,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8269422650337219,
      "rewards/margins": 2.919489860534668,
      "rewards/rejected": -3.7464325428009033,
      "step": 4460
    },
    {
      "epoch": 2.84,
      "learning_rate": 3.0317273795534665e-08,
      "logits/chosen": -2.5089969635009766,
      "logits/rejected": -2.625685930252075,
      "logps/chosen": -235.75051879882812,
      "logps/rejected": -300.0323181152344,
      "loss": 0.1282,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9407854080200195,
      "rewards/margins": 3.251286029815674,
      "rewards/rejected": -4.192071437835693,
      "step": 4470
    },
    {
      "epoch": 2.84,
      "learning_rate": 2.9142185663924792e-08,
      "logits/chosen": -2.4949374198913574,
      "logits/rejected": -2.6092774868011475,
      "logps/chosen": -261.3769836425781,
      "logps/rejected": -353.140869140625,
      "loss": 0.1628,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9599424600601196,
      "rewards/margins": 3.2244582176208496,
      "rewards/rejected": -4.184401035308838,
      "step": 4480
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.7967097532314924e-08,
      "logits/chosen": -2.5348312854766846,
      "logits/rejected": -2.6496498584747314,
      "logps/chosen": -202.01119995117188,
      "logps/rejected": -278.031005859375,
      "loss": 0.1679,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.8432455062866211,
      "rewards/margins": 3.2406005859375,
      "rewards/rejected": -4.083846092224121,
      "step": 4490
    },
    {
      "epoch": 2.86,
      "learning_rate": 2.6792009400705055e-08,
      "logits/chosen": -2.5073933601379395,
      "logits/rejected": -2.6364850997924805,
      "logps/chosen": -289.18939208984375,
      "logps/rejected": -367.5996398925781,
      "loss": 0.1439,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.9730752110481262,
      "rewards/margins": 3.5499134063720703,
      "rewards/rejected": -4.522988796234131,
      "step": 4500
    },
    {
      "epoch": 2.86,
      "eval_logits/chosen": -2.6262805461883545,
      "eval_logits/rejected": -2.7516534328460693,
      "eval_logps/chosen": -280.28082275390625,
      "eval_logps/rejected": -327.82525634765625,
      "eval_loss": 0.17758876085281372,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9367989301681519,
      "eval_rewards/margins": 3.174954414367676,
      "eval_rewards/rejected": -4.111752986907959,
      "eval_runtime": 111.6184,
      "eval_samples_per_second": 2.311,
      "eval_steps_per_second": 0.582,
      "step": 4500
    },
    {
      "epoch": 2.86,
      "learning_rate": 2.561692126909518e-08,
      "logits/chosen": -2.539508104324341,
      "logits/rejected": -2.6655426025390625,
      "logps/chosen": -261.8391418457031,
      "logps/rejected": -326.2175598144531,
      "loss": 0.1739,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8831302523612976,
      "rewards/margins": 3.27866792678833,
      "rewards/rejected": -4.161798477172852,
      "step": 4510
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.444183313748531e-08,
      "logits/chosen": -2.4665322303771973,
      "logits/rejected": -2.5921568870544434,
      "logps/chosen": -225.9403839111328,
      "logps/rejected": -302.77252197265625,
      "loss": 0.1307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8694049119949341,
      "rewards/margins": 3.497511625289917,
      "rewards/rejected": -4.366917133331299,
      "step": 4520
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.326674500587544e-08,
      "logits/chosen": -2.466461420059204,
      "logits/rejected": -2.6197025775909424,
      "logps/chosen": -237.28945922851562,
      "logps/rejected": -303.01922607421875,
      "loss": 0.1546,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8372966647148132,
      "rewards/margins": 3.4156036376953125,
      "rewards/rejected": -4.25290060043335,
      "step": 4530
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.209165687426557e-08,
      "logits/chosen": -2.497443199157715,
      "logits/rejected": -2.630983591079712,
      "logps/chosen": -269.23138427734375,
      "logps/rejected": -361.00927734375,
      "loss": 0.1433,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0092241764068604,
      "rewards/margins": 3.8250317573547363,
      "rewards/rejected": -4.834255695343018,
      "step": 4540
    },
    {
      "epoch": 2.89,
      "learning_rate": 2.09165687426557e-08,
      "logits/chosen": -2.518745183944702,
      "logits/rejected": -2.6550798416137695,
      "logps/chosen": -247.175537109375,
      "logps/rejected": -316.89007568359375,
      "loss": 0.149,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.0203502178192139,
      "rewards/margins": 3.0870394706726074,
      "rewards/rejected": -4.1073899269104,
      "step": 4550
    },
    {
      "epoch": 2.89,
      "learning_rate": 1.9741480611045828e-08,
      "logits/chosen": -2.5766825675964355,
      "logits/rejected": -2.7187819480895996,
      "logps/chosen": -186.9510498046875,
      "logps/rejected": -297.41839599609375,
      "loss": 0.1995,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7331101298332214,
      "rewards/margins": 3.3479042053222656,
      "rewards/rejected": -4.0810136795043945,
      "step": 4560
    },
    {
      "epoch": 2.9,
      "learning_rate": 1.8566392479435956e-08,
      "logits/chosen": -2.5755953788757324,
      "logits/rejected": -2.679940700531006,
      "logps/chosen": -219.3028564453125,
      "logps/rejected": -296.7222595214844,
      "loss": 0.2056,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0375137329101562,
      "rewards/margins": 2.938987970352173,
      "rewards/rejected": -3.976501941680908,
      "step": 4570
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.7391304347826087e-08,
      "logits/chosen": -2.5530171394348145,
      "logits/rejected": -2.6780002117156982,
      "logps/chosen": -261.05126953125,
      "logps/rejected": -344.45660400390625,
      "loss": 0.1465,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.765406608581543,
      "rewards/margins": 3.5452933311462402,
      "rewards/rejected": -4.310700416564941,
      "step": 4580
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.6216216216216218e-08,
      "logits/chosen": -2.4496970176696777,
      "logits/rejected": -2.5862369537353516,
      "logps/chosen": -224.5316925048828,
      "logps/rejected": -318.96746826171875,
      "loss": 0.1395,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8896668553352356,
      "rewards/margins": 3.4528369903564453,
      "rewards/rejected": -4.342503547668457,
      "step": 4590
    },
    {
      "epoch": 2.92,
      "learning_rate": 1.5041128084606346e-08,
      "logits/chosen": -2.5245232582092285,
      "logits/rejected": -2.662123203277588,
      "logps/chosen": -248.1486053466797,
      "logps/rejected": -310.48223876953125,
      "loss": 0.1116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9665654897689819,
      "rewards/margins": 3.5374348163604736,
      "rewards/rejected": -4.504000186920166,
      "step": 4600
    },
    {
      "epoch": 2.92,
      "eval_logits/chosen": -2.627204418182373,
      "eval_logits/rejected": -2.7525758743286133,
      "eval_logps/chosen": -280.2152404785156,
      "eval_logps/rejected": -327.78668212890625,
      "eval_loss": 0.17727039754390717,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9302414655685425,
      "eval_rewards/margins": 3.1776533126831055,
      "eval_rewards/rejected": -4.1078948974609375,
      "eval_runtime": 103.472,
      "eval_samples_per_second": 2.493,
      "eval_steps_per_second": 0.628,
      "step": 4600
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.3866039952996475e-08,
      "logits/chosen": -2.5982613563537598,
      "logits/rejected": -2.7002675533294678,
      "logps/chosen": -264.13037109375,
      "logps/rejected": -314.408935546875,
      "loss": 0.2151,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7936927080154419,
      "rewards/margins": 2.8953773975372314,
      "rewards/rejected": -3.689070224761963,
      "step": 4610
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.2690951821386603e-08,
      "logits/chosen": -2.4765639305114746,
      "logits/rejected": -2.6121089458465576,
      "logps/chosen": -287.32159423828125,
      "logps/rejected": -348.04998779296875,
      "loss": 0.1313,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.1120750904083252,
      "rewards/margins": 3.524439573287964,
      "rewards/rejected": -4.6365156173706055,
      "step": 4620
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.1515863689776732e-08,
      "logits/chosen": -2.5346221923828125,
      "logits/rejected": -2.6657118797302246,
      "logps/chosen": -282.83428955078125,
      "logps/rejected": -348.8053283691406,
      "loss": 0.1698,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7843857407569885,
      "rewards/margins": 3.3374381065368652,
      "rewards/rejected": -4.121823310852051,
      "step": 4630
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0340775558166862e-08,
      "logits/chosen": -2.434760570526123,
      "logits/rejected": -2.554131507873535,
      "logps/chosen": -237.957763671875,
      "logps/rejected": -318.7060546875,
      "loss": 0.1428,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.93940269947052,
      "rewards/margins": 3.30562162399292,
      "rewards/rejected": -4.24502420425415,
      "step": 4640
    },
    {
      "epoch": 2.95,
      "learning_rate": 9.165687426556991e-09,
      "logits/chosen": -2.5313379764556885,
      "logits/rejected": -2.7033989429473877,
      "logps/chosen": -267.8631591796875,
      "logps/rejected": -347.33197021484375,
      "loss": 0.1407,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.9034785032272339,
      "rewards/margins": 3.487234115600586,
      "rewards/rejected": -4.390713214874268,
      "step": 4650
    },
    {
      "epoch": 2.96,
      "learning_rate": 7.99059929494712e-09,
      "logits/chosen": -2.461906909942627,
      "logits/rejected": -2.5894529819488525,
      "logps/chosen": -262.21563720703125,
      "logps/rejected": -343.3765563964844,
      "loss": 0.1208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9875903129577637,
      "rewards/margins": 3.8916656970977783,
      "rewards/rejected": -4.879256248474121,
      "step": 4660
    },
    {
      "epoch": 2.96,
      "learning_rate": 6.81551116333725e-09,
      "logits/chosen": -2.5865933895111084,
      "logits/rejected": -2.6842968463897705,
      "logps/chosen": -246.8555908203125,
      "logps/rejected": -313.49078369140625,
      "loss": 0.1671,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9366054534912109,
      "rewards/margins": 3.3555233478546143,
      "rewards/rejected": -4.292128562927246,
      "step": 4670
    },
    {
      "epoch": 2.97,
      "learning_rate": 5.64042303172738e-09,
      "logits/chosen": -2.4116034507751465,
      "logits/rejected": -2.5966732501983643,
      "logps/chosen": -235.34793090820312,
      "logps/rejected": -343.0075378417969,
      "loss": 0.1324,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7782406210899353,
      "rewards/margins": 3.743253707885742,
      "rewards/rejected": -4.521494388580322,
      "step": 4680
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.465334900117508e-09,
      "logits/chosen": -2.4537487030029297,
      "logits/rejected": -2.6341445446014404,
      "logps/chosen": -227.99148559570312,
      "logps/rejected": -331.6649475097656,
      "loss": 0.1235,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9890607595443726,
      "rewards/margins": 3.8693556785583496,
      "rewards/rejected": -4.858416557312012,
      "step": 4690
    },
    {
      "epoch": 2.98,
      "learning_rate": 3.2902467685076377e-09,
      "logits/chosen": -2.624380588531494,
      "logits/rejected": -2.7386741638183594,
      "logps/chosen": -241.9665069580078,
      "logps/rejected": -295.4974670410156,
      "loss": 0.18,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7609454989433289,
      "rewards/margins": 3.232642412185669,
      "rewards/rejected": -3.9935879707336426,
      "step": 4700
    },
    {
      "epoch": 2.98,
      "eval_logits/chosen": -2.6271278858184814,
      "eval_logits/rejected": -2.7526466846466064,
      "eval_logps/chosen": -280.202880859375,
      "eval_logps/rejected": -327.7554016113281,
      "eval_loss": 0.17721602320671082,
      "eval_rewards/accuracies": 0.8384615182876587,
      "eval_rewards/chosen": -0.9290047287940979,
      "eval_rewards/margins": 3.175762176513672,
      "eval_rewards/rejected": -4.104766845703125,
      "eval_runtime": 101.9956,
      "eval_samples_per_second": 2.53,
      "eval_steps_per_second": 0.637,
      "step": 4700
    },
    {
      "epoch": 2.99,
      "learning_rate": 2.115158636897767e-09,
      "logits/chosen": -2.5959596633911133,
      "logits/rejected": -2.758236885070801,
      "logps/chosen": -254.6003875732422,
      "logps/rejected": -335.0420837402344,
      "loss": 0.1537,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8302834630012512,
      "rewards/margins": 3.368427276611328,
      "rewards/rejected": -4.1987104415893555,
      "step": 4710
    },
    {
      "epoch": 2.99,
      "learning_rate": 9.400705052878966e-10,
      "logits/chosen": -2.493212938308716,
      "logits/rejected": -2.6331095695495605,
      "logps/chosen": -247.1638641357422,
      "logps/rejected": -332.60443115234375,
      "loss": 0.1467,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8931487798690796,
      "rewards/margins": 3.45464825630188,
      "rewards/rejected": -4.34779691696167,
      "step": 4720
    },
    {
      "epoch": 3.0,
      "step": 4728,
      "total_flos": 0.0,
      "train_loss": 0.2459611980317006,
      "train_runtime": 34910.6145,
      "train_samples_per_second": 1.083,
      "train_steps_per_second": 0.135
    }
  ],
  "logging_steps": 10,
  "max_steps": 4728,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 150,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}