{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 625,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.936507936507937e-08,
      "logits/chosen": 0.1349913775920868,
      "logits/rejected": 0.1167084276676178,
      "logps/chosen": -428.1781005859375,
      "logps/rejected": -484.54522705078125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507937e-07,
      "logits/chosen": 0.2796071171760559,
      "logits/rejected": 0.24151457846164703,
      "logps/chosen": -499.6032409667969,
      "logps/rejected": -473.9646301269531,
      "loss": 0.6932,
      "rewards/accuracies": 0.4027777910232544,
      "rewards/chosen": 2.6125486328965053e-05,
      "rewards/margins": 5.101554052089341e-05,
      "rewards/rejected": -2.489005055394955e-05,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015873e-06,
      "logits/chosen": 0.2518273890018463,
      "logits/rejected": 0.32998543977737427,
      "logps/chosen": -489.4015197753906,
      "logps/rejected": -479.5149841308594,
      "loss": 0.6931,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.0016528337728232145,
      "rewards/margins": 0.00016125899855978787,
      "rewards/rejected": -0.001814092742279172,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": 0.18698568642139435,
      "logits/rejected": 0.21475109457969666,
      "logps/chosen": -501.27581787109375,
      "logps/rejected": -522.3826293945312,
      "loss": 0.6927,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.008201382122933865,
      "rewards/margins": 0.0008340775966644287,
      "rewards/rejected": -0.009035459719598293,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031746e-06,
      "logits/chosen": 0.20552173256874084,
      "logits/rejected": 0.18397466838359833,
      "logps/chosen": -533.6796875,
      "logps/rejected": -549.6390380859375,
      "loss": 0.6917,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.024371124804019928,
      "rewards/margins": 0.0025360379368066788,
      "rewards/rejected": -0.026907164603471756,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-06,
      "logits/chosen": 0.2108609676361084,
      "logits/rejected": 0.2548374831676483,
      "logps/chosen": -554.6732177734375,
      "logps/rejected": -545.2399291992188,
      "loss": 0.6896,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0501377210021019,
      "rewards/margins": 0.006772263441234827,
      "rewards/rejected": -0.05690998584032059,
      "step": 50
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": 0.19314441084861755,
      "logits/rejected": 0.27108246088027954,
      "logps/chosen": -606.186767578125,
      "logps/rejected": -608.7874145507812,
      "loss": 0.6876,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0944763794541359,
      "rewards/margins": 0.008413041010499,
      "rewards/rejected": -0.10288941860198975,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998086282661188e-06,
      "logits/chosen": 0.16735275089740753,
      "logits/rejected": 0.17189382016658783,
      "logps/chosen": -552.279296875,
      "logps/rejected": -591.57421875,
      "loss": 0.6852,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.11828555911779404,
      "rewards/margins": 0.01398100983351469,
      "rewards/rejected": -0.1322665959596634,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988720025682995e-06,
      "logits/chosen": 0.10864056646823883,
      "logits/rejected": 0.14530186355113983,
      "logps/chosen": -644.5869140625,
      "logps/rejected": -674.5743408203125,
      "loss": 0.6828,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13946327567100525,
      "rewards/margins": 0.019074594601988792,
      "rewards/rejected": -0.1585378646850586,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9715789537359126e-06,
      "logits/chosen": 0.07046853750944138,
      "logits/rejected": 0.1207747831940651,
      "logps/chosen": -684.7418212890625,
      "logps/rejected": -712.235107421875,
      "loss": 0.6792,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.1731593906879425,
      "rewards/margins": 0.031467147171497345,
      "rewards/rejected": -0.20462656021118164,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946716615897932e-06,
      "logits/chosen": 0.0422443263232708,
      "logits/rejected": 0.07933792471885681,
      "logps/chosen": -741.9517211914062,
      "logps/rejected": -768.1805419921875,
      "loss": 0.6793,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2586411237716675,
      "rewards/margins": 0.030019978061318398,
      "rewards/rejected": -0.28866106271743774,
      "step": 100
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9142106826480114e-06,
      "logits/chosen": 0.01217989344149828,
      "logits/rejected": 0.036614928394556046,
      "logps/chosen": -704.1077880859375,
      "logps/rejected": -766.5753173828125,
      "loss": 0.6816,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.24351103603839874,
      "rewards/margins": 0.025916840881109238,
      "rewards/rejected": -0.2694278359413147,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.874162703221823e-06,
      "logits/chosen": 0.04041505232453346,
      "logits/rejected": 0.0663461834192276,
      "logps/chosen": -756.8778686523438,
      "logps/rejected": -816.00244140625,
      "loss": 0.6726,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.27453333139419556,
      "rewards/margins": 0.04802945256233215,
      "rewards/rejected": -0.3225627541542053,
      "step": 120
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826697788369752e-06,
      "logits/chosen": -0.03461530804634094,
      "logits/rejected": 0.08036826550960541,
      "logps/chosen": -926.38720703125,
      "logps/rejected": -1025.3724365234375,
      "loss": 0.6617,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.422262579202652,
      "rewards/margins": 0.0845782682299614,
      "rewards/rejected": -0.5068408250808716,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7719642195082224e-06,
      "logits/chosen": 0.032650940120220184,
      "logits/rejected": 0.02779592201113701,
      "logps/chosen": -852.9603271484375,
      "logps/rejected": -985.2545166015625,
      "loss": 0.6576,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.36906754970550537,
      "rewards/margins": 0.09497459977865219,
      "rewards/rejected": -0.46404212713241577,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.710132985485355e-06,
      "logits/chosen": 0.05756285786628723,
      "logits/rejected": 0.1693125069141388,
      "logps/chosen": -1006.541015625,
      "logps/rejected": -1214.77587890625,
      "loss": 0.6513,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5491369962692261,
      "rewards/margins": 0.171253964304924,
      "rewards/rejected": -0.7203909158706665,
      "step": 150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.641397248408122e-06,
      "logits/chosen": 0.25534456968307495,
      "logits/rejected": 0.15446144342422485,
      "logps/chosen": -1380.378173828125,
      "logps/rejected": -1509.271728515625,
      "loss": 0.6491,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8604512214660645,
      "rewards/margins": 0.11827781051397324,
      "rewards/rejected": -0.9787289500236511,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5659717401997655e-06,
      "logits/chosen": 0.11406876146793365,
      "logits/rejected": 0.2429320365190506,
      "logps/chosen": -1162.3896484375,
      "logps/rejected": -1300.2310791015625,
      "loss": 0.6593,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6630619168281555,
      "rewards/margins": 0.1277274191379547,
      "rewards/rejected": -0.7907893061637878,
      "step": 170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4840920917726425e-06,
      "logits/chosen": 0.1443721204996109,
      "logits/rejected": 0.24939250946044922,
      "logps/chosen": -1037.646240234375,
      "logps/rejected": -1174.6041259765625,
      "loss": 0.6464,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5330273509025574,
      "rewards/margins": 0.12969453632831573,
      "rewards/rejected": -0.6627219915390015,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396014096912182e-06,
      "logits/chosen": 0.1525859534740448,
      "logits/rejected": 0.20368242263793945,
      "logps/chosen": -1222.385498046875,
      "logps/rejected": -1366.5484619140625,
      "loss": 0.6629,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7235668897628784,
      "rewards/margins": 0.11871180683374405,
      "rewards/rejected": -0.8422788381576538,
      "step": 190
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302012913171584e-06,
      "logits/chosen": 0.20741760730743408,
      "logits/rejected": 0.23669476807117462,
      "logps/chosen": -1129.2685546875,
      "logps/rejected": -1311.441650390625,
      "loss": 0.6541,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.6570686101913452,
      "rewards/margins": 0.16556376218795776,
      "rewards/rejected": -0.8226323127746582,
      "step": 200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.202382202273702e-06,
      "logits/chosen": 0.20800559222698212,
      "logits/rejected": 0.2900869846343994,
      "logps/chosen": -1247.2933349609375,
      "logps/rejected": -1375.472900390625,
      "loss": 0.6643,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6747683882713318,
      "rewards/margins": 0.1478392481803894,
      "rewards/rejected": -0.8226076364517212,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097433212705492e-06,
      "logits/chosen": 0.1988043189048767,
      "logits/rejected": 0.2728360593318939,
      "logps/chosen": -1089.7906494140625,
      "logps/rejected": -1184.677490234375,
      "loss": 0.6588,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.5955246686935425,
      "rewards/margins": 0.07477297633886337,
      "rewards/rejected": -0.6702976226806641,
      "step": 220
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.987493807371033e-06,
      "logits/chosen": 0.2314763069152832,
      "logits/rejected": 0.34903794527053833,
      "logps/chosen": -1126.2254638671875,
      "logps/rejected": -1295.7572021484375,
      "loss": 0.6408,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6263796091079712,
      "rewards/margins": 0.1536393165588379,
      "rewards/rejected": -0.7800189852714539,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.872907439340758e-06,
      "logits/chosen": 0.24221527576446533,
      "logits/rejected": 0.3353860080242157,
      "logps/chosen": -1068.5975341796875,
      "logps/rejected": -1235.1514892578125,
      "loss": 0.6518,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6160915493965149,
      "rewards/margins": 0.1515791118144989,
      "rewards/rejected": -0.7676706910133362,
      "step": 240
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.75403207889666e-06,
      "logits/chosen": 0.2971830368041992,
      "logits/rejected": 0.34285473823547363,
      "logps/chosen": -1286.203369140625,
      "logps/rejected": -1462.2490234375,
      "loss": 0.647,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7703440189361572,
      "rewards/margins": 0.16007237136363983,
      "rewards/rejected": -0.9304164052009583,
      "step": 250
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631239095225417e-06,
      "logits/chosen": 0.22825905680656433,
      "logits/rejected": 0.32187604904174805,
      "logps/chosen": -1112.8629150390625,
      "logps/rejected": -1291.74072265625,
      "loss": 0.64,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.6394481658935547,
      "rewards/margins": 0.16354107856750488,
      "rewards/rejected": -0.8029893040657043,
      "step": 260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5049120962530608e-06,
      "logits/chosen": 0.32051190733909607,
      "logits/rejected": 0.2698783576488495,
      "logps/chosen": -1120.5859375,
      "logps/rejected": -1305.715087890625,
      "loss": 0.6356,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6476856470108032,
      "rewards/margins": 0.15380369126796722,
      "rewards/rejected": -0.8014893531799316,
      "step": 270
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3754457302455464e-06,
      "logits/chosen": 0.23477685451507568,
      "logits/rejected": 0.43997740745544434,
      "logps/chosen": -1300.92724609375,
      "logps/rejected": -1495.2685546875,
      "loss": 0.6537,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.8141768574714661,
      "rewards/margins": 0.17344731092453003,
      "rewards/rejected": -0.9876241683959961,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2432444529190714e-06,
      "logits/chosen": 0.38444215059280396,
      "logits/rejected": 0.43280649185180664,
      "logps/chosen": -1254.042724609375,
      "logps/rejected": -1498.3414306640625,
      "loss": 0.6399,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7942158579826355,
      "rewards/margins": 0.20459000766277313,
      "rewards/rejected": -0.9988058805465698,
      "step": 290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1087212639117057e-06,
      "logits/chosen": 0.35266411304473877,
      "logits/rejected": 0.5088330507278442,
      "logps/chosen": -1342.4091796875,
      "logps/rejected": -1464.3389892578125,
      "loss": 0.6541,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8787577748298645,
      "rewards/margins": 0.11491602659225464,
      "rewards/rejected": -0.9936738014221191,
      "step": 300
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9722964165636263e-06,
      "logits/chosen": 0.3081873953342438,
      "logits/rejected": 0.4749964773654938,
      "logps/chosen": -1241.6292724609375,
      "logps/rejected": -1437.963134765625,
      "loss": 0.6523,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.7512220144271851,
      "rewards/margins": 0.16275373101234436,
      "rewards/rejected": -0.913975715637207,
      "step": 310
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8343961050366275e-06,
      "logits/chosen": 0.3727472424507141,
      "logits/rejected": 0.41282615065574646,
      "logps/chosen": -1198.703857421875,
      "logps/rejected": -1467.189208984375,
      "loss": 0.6399,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.719472348690033,
      "rewards/margins": 0.23017852008342743,
      "rewards/rejected": -0.9496508836746216,
      "step": 320
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.695451132874385e-06,
      "logits/chosen": 0.3753197193145752,
      "logits/rejected": 0.4588402211666107,
      "logps/chosen": -1306.0467529296875,
      "logps/rejected": -1423.9765625,
      "loss": 0.6339,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.821855902671814,
      "rewards/margins": 0.15302011370658875,
      "rewards/rejected": -0.9748760461807251,
      "step": 330
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5558955671628964e-06,
      "logits/chosen": 0.32516294717788696,
      "logits/rejected": 0.49244314432144165,
      "logps/chosen": -1465.662353515625,
      "logps/rejected": -1645.747314453125,
      "loss": 0.6415,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.933737576007843,
      "rewards/margins": 0.19395960867404938,
      "rewards/rejected": -1.127697229385376,
      "step": 340
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4161653824955654e-06,
      "logits/chosen": 0.37509265542030334,
      "logits/rejected": 0.5179189443588257,
      "logps/chosen": -1365.8070068359375,
      "logps/rejected": -1507.181396484375,
      "loss": 0.639,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.868498682975769,
      "rewards/margins": 0.15674063563346863,
      "rewards/rejected": -1.0252392292022705,
      "step": 350
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2766970989791697e-06,
      "logits/chosen": 0.3914629817008972,
      "logits/rejected": 0.39266401529312134,
      "logps/chosen": -1348.240966796875,
      "logps/rejected": -1593.037353515625,
      "loss": 0.6485,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8051446080207825,
      "rewards/margins": 0.2146397829055786,
      "rewards/rejected": -1.0197844505310059,
      "step": 360
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1379264185356545e-06,
      "logits/chosen": 0.29912859201431274,
      "logits/rejected": 0.4203578531742096,
      "logps/chosen": -1122.2073974609375,
      "logps/rejected": -1355.9158935546875,
      "loss": 0.6317,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.7034318447113037,
      "rewards/margins": 0.18991844356060028,
      "rewards/rejected": -0.8933502435684204,
      "step": 370
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.000286863759934e-06,
      "logits/chosen": 0.37296754121780396,
      "logits/rejected": 0.569365382194519,
      "logps/chosen": -1219.0467529296875,
      "logps/rejected": -1441.828857421875,
      "loss": 0.6332,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7400010228157043,
      "rewards/margins": 0.20997154712677002,
      "rewards/rejected": -0.9499725103378296,
      "step": 380
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8642084235859764e-06,
      "logits/chosen": 0.39869484305381775,
      "logits/rejected": 0.49166345596313477,
      "logps/chosen": -1271.091064453125,
      "logps/rejected": -1447.0933837890625,
      "loss": 0.6447,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7757032513618469,
      "rewards/margins": 0.18741454184055328,
      "rewards/rejected": -0.9631179571151733,
      "step": 390
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7301162099921013e-06,
      "logits/chosen": 0.48826533555984497,
      "logits/rejected": 0.5182665586471558,
      "logps/chosen": -1342.3040771484375,
      "logps/rejected": -1515.572021484375,
      "loss": 0.6527,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8140773773193359,
      "rewards/margins": 0.18610136210918427,
      "rewards/rejected": -1.0001786947250366,
      "step": 400
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5984291299420117e-06,
      "logits/chosen": 0.4482479989528656,
      "logits/rejected": 0.494678258895874,
      "logps/chosen": -1258.864501953125,
      "logps/rejected": -1553.4383544921875,
      "loss": 0.6073,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.7780212163925171,
      "rewards/margins": 0.25596219301223755,
      "rewards/rejected": -1.0339832305908203,
      "step": 410
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4695585767104092e-06,
      "logits/chosen": 0.4765148162841797,
      "logits/rejected": 0.44907188415527344,
      "logps/chosen": -1387.763427734375,
      "logps/rejected": -1522.737548828125,
      "loss": 0.6573,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.9270384907722473,
      "rewards/margins": 0.10358323156833649,
      "rewards/rejected": -1.0306217670440674,
      "step": 420
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3439071446815452e-06,
      "logits/chosen": 0.4347519278526306,
      "logits/rejected": 0.5749093294143677,
      "logps/chosen": -1321.0262451171875,
      "logps/rejected": -1519.8450927734375,
      "loss": 0.6397,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8617701530456543,
      "rewards/margins": 0.19662390649318695,
      "rewards/rejected": -1.0583940744400024,
      "step": 430
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2218673716356919e-06,
      "logits/chosen": 0.4529833197593689,
      "logits/rejected": 0.5917688608169556,
      "logps/chosen": -1364.6546630859375,
      "logps/rejected": -1504.9599609375,
      "loss": 0.6405,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8427094221115112,
      "rewards/margins": 0.1536230742931366,
      "rewards/rejected": -0.9963325262069702,
      "step": 440
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.103820512452661e-06,
      "logits/chosen": 0.3574792444705963,
      "logits/rejected": 0.4730028510093689,
      "logps/chosen": -1273.257568359375,
      "logps/rejected": -1469.2452392578125,
      "loss": 0.616,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7781795859336853,
      "rewards/margins": 0.17071351408958435,
      "rewards/rejected": -0.9488931894302368,
      "step": 450
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.901353480633468e-07,
      "logits/chosen": 0.43064388632774353,
      "logits/rejected": 0.6706501245498657,
      "logps/chosen": -1389.450439453125,
      "logps/rejected": -1536.8935546875,
      "loss": 0.6465,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8454073071479797,
      "rewards/margins": 0.16833043098449707,
      "rewards/rejected": -1.0137377977371216,
      "step": 460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.811670333701544e-07,
      "logits/chosen": 0.46944761276245117,
      "logits/rejected": 0.5561131238937378,
      "logps/chosen": -1286.3804931640625,
      "logps/rejected": -1454.6707763671875,
      "loss": 0.6409,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.8223176002502441,
      "rewards/margins": 0.14870892465114594,
      "rewards/rejected": -0.971026599407196,
      "step": 470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.772559877354341e-07,
      "logits/chosen": 0.44347596168518066,
      "logits/rejected": 0.5607708692550659,
      "logps/chosen": -1329.2608642578125,
      "logps/rejected": -1553.534912109375,
      "loss": 0.6401,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8247483968734741,
      "rewards/margins": 0.23265239596366882,
      "rewards/rejected": -1.0574008226394653,
      "step": 480
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787268315040604e-07,
      "logits/chosen": 0.45571979880332947,
      "logits/rejected": 0.47239336371421814,
      "logps/chosen": -1211.712646484375,
      "logps/rejected": -1386.2025146484375,
      "loss": 0.6304,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7387667298316956,
      "rewards/margins": 0.1702551543712616,
      "rewards/rejected": -0.9090218544006348,
      "step": 490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.858873718824829e-07,
      "logits/chosen": 0.3857249319553375,
      "logits/rejected": 0.5553338527679443,
      "logps/chosen": -1262.7962646484375,
      "logps/rejected": -1481.9622802734375,
      "loss": 0.6333,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7554186582565308,
      "rewards/margins": 0.20985445380210876,
      "rewards/rejected": -0.9652730822563171,
      "step": 500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.990276413423817e-07,
      "logits/chosen": 0.4313405454158783,
      "logits/rejected": 0.590368390083313,
      "logps/chosen": -1306.13427734375,
      "logps/rejected": -1447.4407958984375,
      "loss": 0.6303,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7738032341003418,
      "rewards/margins": 0.16366735100746155,
      "rewards/rejected": -0.9374706149101257,
      "step": 510
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.184189915529796e-07,
      "logits/chosen": 0.4014933109283447,
      "logits/rejected": 0.5485681891441345,
      "logps/chosen": -1255.651123046875,
      "logps/rejected": -1364.13232421875,
      "loss": 0.6527,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.7963151931762695,
      "rewards/margins": 0.10994601249694824,
      "rewards/rejected": -0.9062612652778625,
      "step": 520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4431324567258176e-07,
      "logits/chosen": 0.48007622361183167,
      "logits/rejected": 0.5644385814666748,
      "logps/chosen": -1217.1119384765625,
      "logps/rejected": -1428.557861328125,
      "loss": 0.6382,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7597641944885254,
      "rewards/margins": 0.2099626362323761,
      "rewards/rejected": -0.9697268605232239,
      "step": 530
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.769419116476052e-07,
      "logits/chosen": 0.41416773200035095,
      "logits/rejected": 0.5963398218154907,
      "logps/chosen": -1281.758544921875,
      "logps/rejected": -1478.4769287109375,
      "loss": 0.6332,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7958284020423889,
      "rewards/margins": 0.19340744614601135,
      "rewards/rejected": -0.9892358779907227,
      "step": 540
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1651545897676512e-07,
      "logits/chosen": 0.4229779839515686,
      "logits/rejected": 0.6195346117019653,
      "logps/chosen": -1226.513427734375,
      "logps/rejected": -1382.2999267578125,
      "loss": 0.6559,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.76485675573349,
      "rewards/margins": 0.13517045974731445,
      "rewards/rejected": -0.9000272750854492,
      "step": 550
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6322266119983222e-07,
      "logits/chosen": 0.46644434332847595,
      "logits/rejected": 0.5570663213729858,
      "logps/chosen": -1331.478271484375,
      "logps/rejected": -1517.951416015625,
      "loss": 0.6477,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8168514966964722,
      "rewards/margins": 0.17577320337295532,
      "rewards/rejected": -0.9926246404647827,
      "step": 560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1723000616502167e-07,
      "logits/chosen": 0.414206326007843,
      "logits/rejected": 0.6192079782485962,
      "logps/chosen": -1307.5006103515625,
      "logps/rejected": -1502.477783203125,
      "loss": 0.6329,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8042691349983215,
      "rewards/margins": 0.1876949965953827,
      "rewards/rejected": -0.9919641613960266,
      "step": 570
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.868117591737585e-08,
      "logits/chosen": 0.3812820017337799,
      "logits/rejected": 0.6579981446266174,
      "logps/chosen": -1247.164794921875,
      "logps/rejected": -1437.7857666015625,
      "loss": 0.643,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.7581318616867065,
      "rewards/margins": 0.20403996109962463,
      "rewards/rejected": -0.9621719121932983,
      "step": 580
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.769659783295383e-08,
      "logits/chosen": 0.38649970293045044,
      "logits/rejected": 0.5585275888442993,
      "logps/chosen": -1279.88818359375,
      "logps/rejected": -1423.707763671875,
      "loss": 0.6325,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.7771328687667847,
      "rewards/margins": 0.1578211635351181,
      "rewards/rejected": -0.9349539875984192,
      "step": 590
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4373068401120358e-08,
      "logits/chosen": 0.5281786918640137,
      "logits/rejected": 0.5024128556251526,
      "logps/chosen": -1351.9422607421875,
      "logps/rejected": -1575.468505859375,
      "loss": 0.6309,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7931488752365112,
      "rewards/margins": 0.2438853532075882,
      "rewards/rejected": -1.037034273147583,
      "step": 600
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.78345083022425e-09,
      "logits/chosen": 0.42136329412460327,
      "logits/rejected": 0.5088121891021729,
      "logps/chosen": -1306.164794921875,
      "logps/rejected": -1567.685546875,
      "loss": 0.6363,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.8058547973632812,
      "rewards/margins": 0.24808284640312195,
      "rewards/rejected": -1.0539376735687256,
      "step": 610
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.764474213677654e-10,
      "logits/chosen": 0.45875900983810425,
      "logits/rejected": 0.5520389080047607,
      "logps/chosen": -1280.700439453125,
      "logps/rejected": -1439.426025390625,
      "loss": 0.6465,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.7951013445854187,
      "rewards/margins": 0.15834487974643707,
      "rewards/rejected": -0.9534462094306946,
      "step": 620
    },
    {
      "epoch": 1.0,
      "step": 625,
      "total_flos": 0.0,
      "train_loss": 0.3321857452392578,
      "train_runtime": 4002.829,
      "train_samples_per_second": 4.996,
      "train_steps_per_second": 0.156
    }
  ],
  "logging_steps": 10,
  "max_steps": 625,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}