{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 625,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.936507936507937e-08,
      "logits/chosen": 0.24662643671035767,
      "logits/rejected": 0.3199857771396637,
      "logps/chosen": -398.9810791015625,
      "logps/rejected": -445.0414123535156,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507937e-07,
      "logits/chosen": 0.2540830075740814,
      "logits/rejected": 0.28116142749786377,
      "logps/chosen": -492.2332763671875,
      "logps/rejected": -497.7977294921875,
      "loss": 0.6931,
      "rewards/accuracies": 0.3958333432674408,
      "rewards/chosen": -0.0001207808600156568,
      "rewards/margins": -3.325760189909488e-05,
      "rewards/rejected": -8.752326539251953e-05,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015873e-06,
      "logits/chosen": 0.19658036530017853,
      "logits/rejected": 0.304654985666275,
      "logps/chosen": -462.8489685058594,
      "logps/rejected": -485.5413513183594,
      "loss": 0.693,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.001320437528192997,
      "rewards/margins": 0.0002517173998057842,
      "rewards/rejected": -0.001572155044414103,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": 0.233014315366745,
      "logits/rejected": 0.2545226514339447,
      "logps/chosen": -516.826171875,
      "logps/rejected": -515.098876953125,
      "loss": 0.6921,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.008494555950164795,
      "rewards/margins": 0.0018449639901518822,
      "rewards/rejected": -0.010339519940316677,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031746e-06,
      "logits/chosen": 0.17288276553153992,
      "logits/rejected": 0.2692878246307373,
      "logps/chosen": -518.7865600585938,
      "logps/rejected": -552.9195556640625,
      "loss": 0.6906,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.03214402496814728,
      "rewards/margins": 0.004966778215020895,
      "rewards/rejected": -0.03711080551147461,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-06,
      "logits/chosen": 0.2141023576259613,
      "logits/rejected": 0.2770322263240814,
      "logps/chosen": -574.9302978515625,
      "logps/rejected": -587.6199951171875,
      "loss": 0.6899,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07075769454240799,
      "rewards/margins": 0.009018848650157452,
      "rewards/rejected": -0.07977654784917831,
      "step": 50
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": 0.20746545493602753,
      "logits/rejected": 0.22541038691997528,
      "logps/chosen": -621.58935546875,
      "logps/rejected": -625.3094482421875,
      "loss": 0.6895,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.10110817104578018,
      "rewards/margins": 0.006856895051896572,
      "rewards/rejected": -0.10796506702899933,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998086282661188e-06,
      "logits/chosen": 0.2066463679075241,
      "logits/rejected": 0.218001127243042,
      "logps/chosen": -615.7960815429688,
      "logps/rejected": -646.2564697265625,
      "loss": 0.6859,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1264130175113678,
      "rewards/margins": 0.008419327437877655,
      "rewards/rejected": -0.13483233749866486,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988720025682995e-06,
      "logits/chosen": 0.17233335971832275,
      "logits/rejected": 0.15302792191505432,
      "logps/chosen": -658.7843017578125,
      "logps/rejected": -671.2628173828125,
      "loss": 0.6848,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12070327997207642,
      "rewards/margins": 0.018190523609519005,
      "rewards/rejected": -0.13889379799365997,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9715789537359126e-06,
      "logits/chosen": 0.10335955768823624,
      "logits/rejected": 0.1700659692287445,
      "logps/chosen": -580.2098999023438,
      "logps/rejected": -629.610595703125,
      "loss": 0.6797,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14437471330165863,
      "rewards/margins": 0.03023264743387699,
      "rewards/rejected": -0.17460736632347107,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946716615897932e-06,
      "logits/chosen": 0.08332869410514832,
      "logits/rejected": 0.09228050708770752,
      "logps/chosen": -648.9441528320312,
      "logps/rejected": -705.8828735351562,
      "loss": 0.6796,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.19636474549770355,
      "rewards/margins": 0.02650091052055359,
      "rewards/rejected": -0.22286565601825714,
      "step": 100
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9142106826480114e-06,
      "logits/chosen": 0.01861090026795864,
      "logits/rejected": 0.041032858192920685,
      "logps/chosen": -754.0704956054688,
      "logps/rejected": -777.1426391601562,
      "loss": 0.6776,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.27138790488243103,
      "rewards/margins": 0.0310644693672657,
      "rewards/rejected": -0.3024523854255676,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.874162703221823e-06,
      "logits/chosen": -0.012570058926939964,
      "logits/rejected": -0.028305906802415848,
      "logps/chosen": -888.3677978515625,
      "logps/rejected": -892.1744384765625,
      "loss": 0.6705,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.34433773159980774,
      "rewards/margins": 0.03458678722381592,
      "rewards/rejected": -0.3789244592189789,
      "step": 120
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826697788369752e-06,
      "logits/chosen": -0.039124295115470886,
      "logits/rejected": 0.005070955958217382,
      "logps/chosen": -856.4949340820312,
      "logps/rejected": -902.6595458984375,
      "loss": 0.6731,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3688521981239319,
      "rewards/margins": 0.05119430273771286,
      "rewards/rejected": -0.42004650831222534,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7719642195082224e-06,
      "logits/chosen": -0.051692504435777664,
      "logits/rejected": 0.06696543842554092,
      "logps/chosen": -899.3610229492188,
      "logps/rejected": -987.4519653320312,
      "loss": 0.6609,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.38903677463531494,
      "rewards/margins": 0.10688074678182602,
      "rewards/rejected": -0.49591749906539917,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.710132985485355e-06,
      "logits/chosen": 0.04679523780941963,
      "logits/rejected": 0.16161568462848663,
      "logps/chosen": -1067.924072265625,
      "logps/rejected": -1131.458984375,
      "loss": 0.6745,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5985130071640015,
      "rewards/margins": 0.07710713148117065,
      "rewards/rejected": -0.6756200790405273,
      "step": 150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.641397248408122e-06,
      "logits/chosen": 0.12509503960609436,
      "logits/rejected": 0.20963266491889954,
      "logps/chosen": -970.3092651367188,
      "logps/rejected": -1117.892333984375,
      "loss": 0.6506,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4672892987728119,
      "rewards/margins": 0.15404292941093445,
      "rewards/rejected": -0.6213322877883911,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5659717401997655e-06,
      "logits/chosen": 0.1800670623779297,
      "logits/rejected": 0.20961984992027283,
      "logps/chosen": -1060.942626953125,
      "logps/rejected": -1264.063232421875,
      "loss": 0.6553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5244041681289673,
      "rewards/margins": 0.17471647262573242,
      "rewards/rejected": -0.6991206407546997,
      "step": 170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4840920917726425e-06,
      "logits/chosen": 0.25674110651016235,
      "logits/rejected": 0.2803070843219757,
      "logps/chosen": -1182.87060546875,
      "logps/rejected": -1315.55712890625,
      "loss": 0.6543,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.7135756611824036,
      "rewards/margins": 0.13786718249320984,
      "rewards/rejected": -0.851442813873291,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396014096912182e-06,
      "logits/chosen": 0.1727183610200882,
      "logits/rejected": 0.35109418630599976,
      "logps/chosen": -1045.9290771484375,
      "logps/rejected": -1137.6866455078125,
      "loss": 0.6689,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5587235689163208,
      "rewards/margins": 0.10695569217205048,
      "rewards/rejected": -0.6656793355941772,
      "step": 190
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302012913171584e-06,
      "logits/chosen": 0.21685247123241425,
      "logits/rejected": 0.1864197552204132,
      "logps/chosen": -865.9853515625,
      "logps/rejected": -946.7151489257812,
      "loss": 0.6622,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3805222511291504,
      "rewards/margins": 0.08247803151607513,
      "rewards/rejected": -0.46300023794174194,
      "step": 200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.202382202273702e-06,
      "logits/chosen": 0.3170011639595032,
      "logits/rejected": 0.26561760902404785,
      "logps/chosen": -860.9000854492188,
      "logps/rejected": -995.12353515625,
      "loss": 0.6505,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4116226136684418,
      "rewards/margins": 0.10268483310937881,
      "rewards/rejected": -0.5143074989318848,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097433212705492e-06,
      "logits/chosen": 0.32917365431785583,
      "logits/rejected": 0.422170490026474,
      "logps/chosen": -1080.322021484375,
      "logps/rejected": -1246.759521484375,
      "loss": 0.6454,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.6017876863479614,
      "rewards/margins": 0.16769546270370483,
      "rewards/rejected": -0.7694831490516663,
      "step": 220
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.987493807371033e-06,
      "logits/chosen": 0.3952299654483795,
      "logits/rejected": 0.39465999603271484,
      "logps/chosen": -1251.5406494140625,
      "logps/rejected": -1388.2440185546875,
      "loss": 0.6442,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.7216378450393677,
      "rewards/margins": 0.1245543584227562,
      "rewards/rejected": -0.8461921811103821,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.872907439340758e-06,
      "logits/chosen": 0.4063127934932709,
      "logits/rejected": 0.47256404161453247,
      "logps/chosen": -1212.349853515625,
      "logps/rejected": -1397.411865234375,
      "loss": 0.6451,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7250560522079468,
      "rewards/margins": 0.1564178168773651,
      "rewards/rejected": -0.8814738392829895,
      "step": 240
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.75403207889666e-06,
      "logits/chosen": 0.3914504051208496,
      "logits/rejected": 0.3554847240447998,
      "logps/chosen": -1249.439208984375,
      "logps/rejected": -1423.4649658203125,
      "loss": 0.6436,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7152602076530457,
      "rewards/margins": 0.17244547605514526,
      "rewards/rejected": -0.8877056837081909,
      "step": 250
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631239095225417e-06,
      "logits/chosen": 0.3514712452888489,
      "logits/rejected": 0.45444172620773315,
      "logps/chosen": -1238.7828369140625,
      "logps/rejected": -1391.1258544921875,
      "loss": 0.6492,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7164724469184875,
      "rewards/margins": 0.16447243094444275,
      "rewards/rejected": -0.8809448480606079,
      "step": 260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5049120962530608e-06,
      "logits/chosen": 0.375350683927536,
      "logits/rejected": 0.4519611895084381,
      "logps/chosen": -1169.996337890625,
      "logps/rejected": -1291.4034423828125,
      "loss": 0.6529,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6447538137435913,
      "rewards/margins": 0.12557630240917206,
      "rewards/rejected": -0.7703300714492798,
      "step": 270
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3754457302455464e-06,
      "logits/chosen": 0.3003133237361908,
      "logits/rejected": 0.448310911655426,
      "logps/chosen": -1107.552734375,
      "logps/rejected": -1246.8009033203125,
      "loss": 0.6489,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.6071010828018188,
      "rewards/margins": 0.13486045598983765,
      "rewards/rejected": -0.7419615387916565,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2432444529190714e-06,
      "logits/chosen": 0.3579423427581787,
      "logits/rejected": 0.432018905878067,
      "logps/chosen": -1139.100830078125,
      "logps/rejected": -1201.1663818359375,
      "loss": 0.6495,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.6468966007232666,
      "rewards/margins": 0.0843476727604866,
      "rewards/rejected": -0.7312443852424622,
      "step": 290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1087212639117057e-06,
      "logits/chosen": 0.3242015540599823,
      "logits/rejected": 0.4742099642753601,
      "logps/chosen": -1214.9383544921875,
      "logps/rejected": -1312.9287109375,
      "loss": 0.6498,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.6838893294334412,
      "rewards/margins": 0.11757048219442368,
      "rewards/rejected": -0.801459789276123,
      "step": 300
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9722964165636263e-06,
      "logits/chosen": 0.4161956310272217,
      "logits/rejected": 0.47839951515197754,
      "logps/chosen": -1124.3660888671875,
      "logps/rejected": -1247.5843505859375,
      "loss": 0.6398,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.6321052312850952,
      "rewards/margins": 0.10825280100107193,
      "rewards/rejected": -0.7403579950332642,
      "step": 310
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8343961050366275e-06,
      "logits/chosen": 0.46386805176734924,
      "logits/rejected": 0.5212578773498535,
      "logps/chosen": -1118.0859375,
      "logps/rejected": -1206.055908203125,
      "loss": 0.6473,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.6330564022064209,
      "rewards/margins": 0.102098748087883,
      "rewards/rejected": -0.7351552248001099,
      "step": 320
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.695451132874385e-06,
      "logits/chosen": 0.5020041465759277,
      "logits/rejected": 0.4887539744377136,
      "logps/chosen": -1234.7464599609375,
      "logps/rejected": -1376.104736328125,
      "loss": 0.6393,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.7160638570785522,
      "rewards/margins": 0.1427764594554901,
      "rewards/rejected": -0.85884028673172,
      "step": 330
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5558955671628964e-06,
      "logits/chosen": 0.44761380553245544,
      "logits/rejected": 0.5008188486099243,
      "logps/chosen": -1225.663818359375,
      "logps/rejected": -1363.5836181640625,
      "loss": 0.6549,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.7390362620353699,
      "rewards/margins": 0.1303921490907669,
      "rewards/rejected": -0.8694285154342651,
      "step": 340
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4161653824955654e-06,
      "logits/chosen": 0.502762496471405,
      "logits/rejected": 0.521251380443573,
      "logps/chosen": -1289.813720703125,
      "logps/rejected": -1509.394775390625,
      "loss": 0.6337,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7973452806472778,
      "rewards/margins": 0.20352289080619812,
      "rewards/rejected": -1.0008680820465088,
      "step": 350
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2766970989791697e-06,
      "logits/chosen": 0.5564379096031189,
      "logits/rejected": 0.4828278422355652,
      "logps/chosen": -1317.9136962890625,
      "logps/rejected": -1490.851806640625,
      "loss": 0.6489,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8256872296333313,
      "rewards/margins": 0.14507783949375153,
      "rewards/rejected": -0.9707651138305664,
      "step": 360
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1379264185356545e-06,
      "logits/chosen": 0.4197818338871002,
      "logits/rejected": 0.4475874900817871,
      "logps/chosen": -1211.713623046875,
      "logps/rejected": -1361.810791015625,
      "loss": 0.6479,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7186134457588196,
      "rewards/margins": 0.15284962952136993,
      "rewards/rejected": -0.8714631199836731,
      "step": 370
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.000286863759934e-06,
      "logits/chosen": 0.4016849398612976,
      "logits/rejected": 0.6099621057510376,
      "logps/chosen": -1292.288818359375,
      "logps/rejected": -1514.7738037109375,
      "loss": 0.6132,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8104116320610046,
      "rewards/margins": 0.20091453194618225,
      "rewards/rejected": -1.0113261938095093,
      "step": 380
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8642084235859764e-06,
      "logits/chosen": 0.551924467086792,
      "logits/rejected": 0.6596301198005676,
      "logps/chosen": -1387.1436767578125,
      "logps/rejected": -1532.2877197265625,
      "loss": 0.6588,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.913176417350769,
      "rewards/margins": 0.1407497525215149,
      "rewards/rejected": -1.0539262294769287,
      "step": 390
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7301162099921013e-06,
      "logits/chosen": 0.5529540777206421,
      "logits/rejected": 0.5772652626037598,
      "logps/chosen": -1380.193115234375,
      "logps/rejected": -1582.646484375,
      "loss": 0.6341,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.8533477783203125,
      "rewards/margins": 0.18637995421886444,
      "rewards/rejected": -1.0397276878356934,
      "step": 400
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5984291299420117e-06,
      "logits/chosen": 0.47877341508865356,
      "logits/rejected": 0.5813654661178589,
      "logps/chosen": -1256.6849365234375,
      "logps/rejected": -1472.9185791015625,
      "loss": 0.6404,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7804775238037109,
      "rewards/margins": 0.19573338329792023,
      "rewards/rejected": -0.97621089220047,
      "step": 410
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4695585767104092e-06,
      "logits/chosen": 0.41678476333618164,
      "logits/rejected": 0.6133105754852295,
      "logps/chosen": -1238.5712890625,
      "logps/rejected": -1406.7432861328125,
      "loss": 0.6409,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7545350193977356,
      "rewards/margins": 0.1725122034549713,
      "rewards/rejected": -0.9270472526550293,
      "step": 420
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3439071446815452e-06,
      "logits/chosen": 0.4347582459449768,
      "logits/rejected": 0.5867087244987488,
      "logps/chosen": -1281.6087646484375,
      "logps/rejected": -1513.9439697265625,
      "loss": 0.6387,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.7874932289123535,
      "rewards/margins": 0.2120811492204666,
      "rewards/rejected": -0.9995743632316589,
      "step": 430
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2218673716356919e-06,
      "logits/chosen": 0.5029743909835815,
      "logits/rejected": 0.591041088104248,
      "logps/chosen": -1364.179931640625,
      "logps/rejected": -1468.728759765625,
      "loss": 0.6623,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8501272201538086,
      "rewards/margins": 0.09424228221178055,
      "rewards/rejected": -0.9443694949150085,
      "step": 440
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.103820512452661e-06,
      "logits/chosen": 0.53369140625,
      "logits/rejected": 0.5491160154342651,
      "logps/chosen": -1355.01171875,
      "logps/rejected": -1442.1351318359375,
      "loss": 0.6686,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.8647180795669556,
      "rewards/margins": 0.09323947131633759,
      "rewards/rejected": -0.9579576253890991,
      "step": 450
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.901353480633468e-07,
      "logits/chosen": 0.5199618339538574,
      "logits/rejected": 0.4953738749027252,
      "logps/chosen": -1270.599853515625,
      "logps/rejected": -1456.319580078125,
      "loss": 0.6241,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7807380557060242,
      "rewards/margins": 0.18214623630046844,
      "rewards/rejected": -0.9628843069076538,
      "step": 460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.811670333701544e-07,
      "logits/chosen": 0.5168347954750061,
      "logits/rejected": 0.6290295124053955,
      "logps/chosen": -1292.0872802734375,
      "logps/rejected": -1423.704345703125,
      "loss": 0.639,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.771618664264679,
      "rewards/margins": 0.1294311136007309,
      "rewards/rejected": -0.901049792766571,
      "step": 470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.772559877354341e-07,
      "logits/chosen": 0.5879518985748291,
      "logits/rejected": 0.6285650730133057,
      "logps/chosen": -1255.3343505859375,
      "logps/rejected": -1419.4129638671875,
      "loss": 0.6316,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8007827997207642,
      "rewards/margins": 0.14159244298934937,
      "rewards/rejected": -0.9423751831054688,
      "step": 480
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787268315040604e-07,
      "logits/chosen": 0.45197635889053345,
      "logits/rejected": 0.5737031698226929,
      "logps/chosen": -1276.957275390625,
      "logps/rejected": -1509.0966796875,
      "loss": 0.6336,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8125177621841431,
      "rewards/margins": 0.19976083934307098,
      "rewards/rejected": -1.012278437614441,
      "step": 490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.858873718824829e-07,
      "logits/chosen": 0.5449025630950928,
      "logits/rejected": 0.6064163446426392,
      "logps/chosen": -1345.1566162109375,
      "logps/rejected": -1465.4927978515625,
      "loss": 0.6838,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.8671814799308777,
      "rewards/margins": 0.12205106019973755,
      "rewards/rejected": -0.9892325401306152,
      "step": 500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.990276413423817e-07,
      "logits/chosen": 0.45466941595077515,
      "logits/rejected": 0.5977617502212524,
      "logps/chosen": -1280.716064453125,
      "logps/rejected": -1477.654541015625,
      "loss": 0.6267,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7988293766975403,
      "rewards/margins": 0.19528616964817047,
      "rewards/rejected": -0.9941156506538391,
      "step": 510
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.184189915529796e-07,
      "logits/chosen": 0.5602769255638123,
      "logits/rejected": 0.611350953578949,
      "logps/chosen": -1306.686279296875,
      "logps/rejected": -1565.746826171875,
      "loss": 0.6323,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.794352650642395,
      "rewards/margins": 0.22704383730888367,
      "rewards/rejected": -1.0213963985443115,
      "step": 520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4431324567258176e-07,
      "logits/chosen": 0.5258222222328186,
      "logits/rejected": 0.5747347474098206,
      "logps/chosen": -1296.38916015625,
      "logps/rejected": -1511.482666015625,
      "loss": 0.6341,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7807685136795044,
      "rewards/margins": 0.19532978534698486,
      "rewards/rejected": -0.9760982394218445,
      "step": 530
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.769419116476052e-07,
      "logits/chosen": 0.46619558334350586,
      "logits/rejected": 0.6566539406776428,
      "logps/chosen": -1344.41748046875,
      "logps/rejected": -1466.8358154296875,
      "loss": 0.6556,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8279752731323242,
      "rewards/margins": 0.1466597020626068,
      "rewards/rejected": -0.9746350049972534,
      "step": 540
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1651545897676512e-07,
      "logits/chosen": 0.5445398092269897,
      "logits/rejected": 0.5394152998924255,
      "logps/chosen": -1226.078125,
      "logps/rejected": -1361.6815185546875,
      "loss": 0.6609,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.7584909200668335,
      "rewards/margins": 0.11591245234012604,
      "rewards/rejected": -0.8744033575057983,
      "step": 550
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6322266119983222e-07,
      "logits/chosen": 0.5224272012710571,
      "logits/rejected": 0.6026440858840942,
      "logps/chosen": -1239.085693359375,
      "logps/rejected": -1359.008544921875,
      "loss": 0.6537,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7636959552764893,
      "rewards/margins": 0.1270333230495453,
      "rewards/rejected": -0.8907292485237122,
      "step": 560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1723000616502167e-07,
      "logits/chosen": 0.44439181685447693,
      "logits/rejected": 0.587325930595398,
      "logps/chosen": -1246.112548828125,
      "logps/rejected": -1382.4180908203125,
      "loss": 0.6502,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.7664066553115845,
      "rewards/margins": 0.1451866328716278,
      "rewards/rejected": -0.9115932583808899,
      "step": 570
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.868117591737585e-08,
      "logits/chosen": 0.5547848343849182,
      "logits/rejected": 0.6335003972053528,
      "logps/chosen": -1389.32470703125,
      "logps/rejected": -1530.272705078125,
      "loss": 0.6396,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8785858154296875,
      "rewards/margins": 0.14252634346485138,
      "rewards/rejected": -1.0211122035980225,
      "step": 580
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.769659783295383e-08,
      "logits/chosen": 0.5736573338508606,
      "logits/rejected": 0.6307505369186401,
      "logps/chosen": -1240.878662109375,
      "logps/rejected": -1390.9312744140625,
      "loss": 0.6415,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8237506747245789,
      "rewards/margins": 0.1187644749879837,
      "rewards/rejected": -0.9425151944160461,
      "step": 590
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4373068401120358e-08,
      "logits/chosen": 0.4911600947380066,
      "logits/rejected": 0.6507722735404968,
      "logps/chosen": -1280.173095703125,
      "logps/rejected": -1408.863037109375,
      "loss": 0.6423,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.7916342616081238,
      "rewards/margins": 0.13446274399757385,
      "rewards/rejected": -0.92609703540802,
      "step": 600
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.78345083022425e-09,
      "logits/chosen": 0.438457727432251,
      "logits/rejected": 0.5983772277832031,
      "logps/chosen": -1361.9815673828125,
      "logps/rejected": -1418.54541015625,
      "loss": 0.6503,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.8475711941719055,
      "rewards/margins": 0.06368283927440643,
      "rewards/rejected": -0.9112540483474731,
      "step": 610
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.764474213677654e-10,
      "logits/chosen": 0.4755234122276306,
      "logits/rejected": 0.6281898617744446,
      "logps/chosen": -1357.261962890625,
      "logps/rejected": -1532.4453125,
      "loss": 0.6579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8383301496505737,
      "rewards/margins": 0.17498043179512024,
      "rewards/rejected": -1.0133105516433716,
      "step": 620
    },
    {
      "epoch": 1.0,
      "step": 625,
      "total_flos": 0.0,
      "train_loss": 0.6552388866424561,
      "train_runtime": 7636.4902,
      "train_samples_per_second": 2.619,
      "train_steps_per_second": 0.082
    }
  ],
  "logging_steps": 10,
  "max_steps": 625,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}