diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,2969 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.988190836088805,
+  "eval_steps": 50,
+  "global_step": 880,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "dpo_loss": 0.6931471824645996,
+      "epoch": 0.005668398677373642,
+      "grad_norm": 134.13619593724954,
+      "learning_rate": 5.681818181818182e-08,
+      "logits": -1.3147305250167847,
+      "logps": -88.0877456665039,
+      "loss": 0.4113,
+      "objective": 0.41588976979255676,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.41588976979255676,
+      "step": 1
+    },
+    {
+      "dpo_loss": 0.693040668964386,
+      "epoch": 0.02834199338686821,
+      "grad_norm": 130.40196272454386,
+      "learning_rate": 2.840909090909091e-07,
+      "logits": -1.3680235147476196,
+      "logps": -84.29497528076172,
+      "loss": 0.412,
+      "objective": 0.3764810860157013,
+      "ranking_idealized": 0.5677083134651184,
+      "ranking_idealized_expo": 0.546875,
+      "ranking_simple": 0.546875,
+      "regularize": 0.3764810860157013,
+      "step": 5
+    },
+    {
+      "dpo_loss": 0.6761094927787781,
+      "epoch": 0.05668398677373642,
+      "grad_norm": 220.67402488716627,
+      "learning_rate": 5.681818181818182e-07,
+      "logits": -1.4485151767730713,
+      "logps": -83.00025177001953,
+      "loss": 0.4272,
+      "objective": 0.4414474666118622,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.4414474666118622,
+      "step": 10
+    },
+    {
+      "dpo_loss": 0.6819472908973694,
+      "epoch": 0.08502598016060463,
+      "grad_norm": 123.4201155289935,
+      "learning_rate": 8.522727272727273e-07,
+      "logits": -1.4201914072036743,
+      "logps": -83.36089324951172,
+      "loss": 0.4193,
+      "objective": 0.41622886061668396,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.41622886061668396,
+      "step": 15
+    },
+    {
+      "dpo_loss": 0.6776646375656128,
+      "epoch": 0.11336797354747284,
+      "grad_norm": 139.10378037423342,
+      "learning_rate": 1.1363636363636364e-06,
+      "logits": -1.4012433290481567,
+      "logps": -84.29528045654297,
+      "loss": 0.442,
+      "objective": 0.45644310116767883,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.45644310116767883,
+      "step": 20
+    },
+    {
+      "dpo_loss": 0.6203070878982544,
+      "epoch": 0.14170996693434104,
+      "grad_norm": 135.72754480990383,
+      "learning_rate": 1.4204545454545458e-06,
+      "logits": -1.4394680261611938,
+      "logps": -84.2686996459961,
+      "loss": 0.4742,
+      "objective": 0.4576638340950012,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5,
+      "regularize": 0.4576638340950012,
+      "step": 25
+    },
+    {
+      "dpo_loss": 0.6442328691482544,
+      "epoch": 0.17005196032120926,
+      "grad_norm": 129.86543736019303,
+      "learning_rate": 1.7045454545454546e-06,
+      "logits": -1.406640887260437,
+      "logps": -84.90333557128906,
+      "loss": 0.5291,
+      "objective": 0.5116989016532898,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.5116989016532898,
+      "step": 30
+    },
+    {
+      "dpo_loss": 0.6669540405273438,
+      "epoch": 0.19839395370807747,
+      "grad_norm": 149.40597914469663,
+      "learning_rate": 1.9886363636363638e-06,
+      "logits": -1.3766274452209473,
+      "logps": -83.32654571533203,
+      "loss": 0.6031,
+      "objective": 0.6114931702613831,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 0.6114931702613831,
+      "step": 35
+    },
+    {
+      "dpo_loss": 0.6527742743492126,
+      "epoch": 0.22673594709494568,
+      "grad_norm": 158.6310433342775,
+      "learning_rate": 2.2727272727272728e-06,
+      "logits": -1.3420394659042358,
+      "logps": -83.61425018310547,
+      "loss": 0.6378,
+      "objective": 0.639444887638092,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.639444887638092,
+      "step": 40
+    },
+    {
+      "dpo_loss": 0.6698108911514282,
+      "epoch": 0.25507794048181387,
+      "grad_norm": 129.09910758768564,
+      "learning_rate": 2.556818181818182e-06,
+      "logits": -1.4372307062149048,
+      "logps": -85.24830627441406,
+      "loss": 0.7526,
+      "objective": 0.8735028505325317,
+      "ranking_idealized": 0.5625,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.8735028505325317,
+      "step": 45
+    },
+    {
+      "dpo_loss": 0.681568443775177,
+      "epoch": 0.2834199338686821,
+      "grad_norm": 118.24660372299884,
+      "learning_rate": 2.8409090909090916e-06,
+      "logits": -1.30116605758667,
+      "logps": -84.97547912597656,
+      "loss": 0.8081,
+      "objective": 0.7729283571243286,
+      "ranking_idealized": 0.4416666626930237,
+      "ranking_idealized_expo": 0.42500001192092896,
+      "ranking_simple": 0.4375,
+      "regularize": 0.7729283571243286,
+      "step": 50
+    },
+    {
+      "epoch": 0.2834199338686821,
+      "eval_dpo_loss": 0.7266114950180054,
+      "eval_logits": -1.330773115158081,
+      "eval_logps": -91.93638610839844,
+      "eval_loss": 0.665158212184906,
+      "eval_objective": 0.6722058653831482,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5123966932296753,
+      "eval_regularize": 0.6722058653831482,
+      "eval_runtime": 260.1252,
+      "eval_samples_per_second": 22.259,
+      "eval_steps_per_second": 0.93,
+      "step": 50
+    },
+    {
+      "dpo_loss": 0.7984517812728882,
+      "epoch": 0.3117619272555503,
+      "grad_norm": 113.66354191682501,
+      "learning_rate": 3.125e-06,
+      "logits": -1.3412364721298218,
+      "logps": -85.0008316040039,
+      "loss": 0.966,
+      "objective": 0.9933412075042725,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.9933412075042725,
+      "step": 55
+    },
+    {
+      "dpo_loss": 0.686260461807251,
+      "epoch": 0.3401039206424185,
+      "grad_norm": 120.77585128117505,
+      "learning_rate": 3.409090909090909e-06,
+      "logits": -1.278722882270813,
+      "logps": -82.67823791503906,
+      "loss": 0.9965,
+      "objective": 0.994083046913147,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.994083046913147,
+      "step": 60
+    },
+    {
+      "dpo_loss": 0.6437157988548279,
+      "epoch": 0.3684459140292867,
+      "grad_norm": 114.63196029717214,
+      "learning_rate": 3.6931818181818186e-06,
+      "logits": -1.1975092887878418,
+      "logps": -80.52145385742188,
+      "loss": 1.0631,
+      "objective": 1.0634738206863403,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 1.0634738206863403,
+      "step": 65
+    },
+    {
+      "dpo_loss": 0.7649748921394348,
+      "epoch": 0.39678790741615494,
+      "grad_norm": 120.79825029146289,
+      "learning_rate": 3.9772727272727275e-06,
+      "logits": -1.2268587350845337,
+      "logps": -81.57954406738281,
+      "loss": 1.1744,
+      "objective": 1.216639757156372,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5,
+      "regularize": 1.216639757156372,
+      "step": 70
+    },
+    {
+      "dpo_loss": 0.8916628360748291,
+      "epoch": 0.42512990080302315,
+      "grad_norm": 94.61834145368111,
+      "learning_rate": 4.2613636363636365e-06,
+      "logits": -1.2359107732772827,
+      "logps": -80.95670318603516,
+      "loss": 1.2475,
+      "objective": 1.3055096864700317,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.5708333253860474,
+      "regularize": 1.3055096864700317,
+      "step": 75
+    },
+    {
+      "dpo_loss": 0.8090317845344543,
+      "epoch": 0.45347189418989137,
+      "grad_norm": 90.2013102285757,
+      "learning_rate": 4.5454545454545455e-06,
+      "logits": -1.2852014303207397,
+      "logps": -79.4792251586914,
+      "loss": 1.2963,
+      "objective": 1.3557077646255493,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5249999761581421,
+      "regularize": 1.3557077646255493,
+      "step": 80
+    },
+    {
+      "dpo_loss": 0.8598435521125793,
+      "epoch": 0.4818138875767596,
+      "grad_norm": 88.10607117738199,
+      "learning_rate": 4.829545454545455e-06,
+      "logits": -1.1217026710510254,
+      "logps": -78.83904266357422,
+      "loss": 1.3155,
+      "objective": 1.3455144166946411,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 1.3455144166946411,
+      "step": 85
+    },
+    {
+      "dpo_loss": 0.9262712001800537,
+      "epoch": 0.5101558809636277,
+      "grad_norm": 91.01793262969673,
+      "learning_rate": 4.999921328558333e-06,
+      "logits": -1.0270463228225708,
+      "logps": -78.34434509277344,
+      "loss": 1.3316,
+      "objective": 1.2936842441558838,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.46666666865348816,
+      "regularize": 1.2936842441558838,
+      "step": 90
+    },
+    {
+      "dpo_loss": 0.8197596073150635,
+      "epoch": 0.538497874350496,
+      "grad_norm": 88.9287273472026,
+      "learning_rate": 4.999036331701828e-06,
+      "logits": -1.1374804973602295,
+      "logps": -75.8565902709961,
+      "loss": 1.4668,
+      "objective": 1.495007038116455,
+      "ranking_idealized": 0.5625,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 1.495007038116455,
+      "step": 95
+    },
+    {
+      "dpo_loss": 0.9542521834373474,
+      "epoch": 0.5668398677373642,
+      "grad_norm": 92.9939055097423,
+      "learning_rate": 4.997168347957521e-06,
+      "logits": -1.1251710653305054,
+      "logps": -78.35294342041016,
+      "loss": 1.4482,
+      "objective": 1.6726794242858887,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5625,
+      "regularize": 1.6726794242858887,
+      "step": 100
+    },
+    {
+      "epoch": 0.5668398677373642,
+      "eval_dpo_loss": 0.9744860529899597,
+      "eval_logits": -1.087980031967163,
+      "eval_logps": -83.3250503540039,
+      "eval_loss": 1.4160186052322388,
+      "eval_objective": 1.3662116527557373,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5092975497245789,
+      "eval_regularize": 1.3662116527557373,
+      "eval_runtime": 258.3781,
+      "eval_samples_per_second": 22.409,
+      "eval_steps_per_second": 0.937,
+      "step": 100
+    },
+    {
+      "dpo_loss": 1.0364277362823486,
+      "epoch": 0.5951818611242324,
+      "grad_norm": 95.45527468177522,
+      "learning_rate": 4.994318112090048e-06,
+      "logits": -0.8985040187835693,
+      "logps": -79.13915252685547,
+      "loss": 1.6082,
+      "objective": 1.6931790113449097,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 1.6931790113449097,
+      "step": 105
+    },
+    {
+      "dpo_loss": 0.9066151976585388,
+      "epoch": 0.6235238545111006,
+      "grad_norm": 82.78277978571477,
+      "learning_rate": 4.990486745229364e-06,
+      "logits": -1.0420721769332886,
+      "logps": -75.80839538574219,
+      "loss": 1.6336,
+      "objective": 1.8092875480651855,
+      "ranking_idealized": 0.47083333134651184,
+      "ranking_idealized_expo": 0.44999998807907104,
+      "ranking_simple": 0.48750001192092896,
+      "regularize": 1.8092875480651855,
+      "step": 110
+    },
+    {
+      "dpo_loss": 1.0691540241241455,
+      "epoch": 0.6518658478979689,
+      "grad_norm": 83.71556017238615,
+      "learning_rate": 4.985675754429744e-06,
+      "logits": -1.0556254386901855,
+      "logps": -75.40866088867188,
+      "loss": 1.6395,
+      "objective": 1.7828887701034546,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.4958333373069763,
+      "regularize": 1.7828887701034546,
+      "step": 115
+    },
+    {
+      "dpo_loss": 1.0725338459014893,
+      "epoch": 0.680207841284837,
+      "grad_norm": 75.45511417337472,
+      "learning_rate": 4.9798870320769884e-06,
+      "logits": -0.9715719819068909,
+      "logps": -74.33720397949219,
+      "loss": 1.5808,
+      "objective": 1.6460652351379395,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 1.6460652351379395,
+      "step": 120
+    },
+    {
+      "dpo_loss": 0.9674696326255798,
+      "epoch": 0.7085498346717053,
+      "grad_norm": 81.29845851322025,
+      "learning_rate": 4.973122855144066e-06,
+      "logits": -0.935053288936615,
+      "logps": -72.40234375,
+      "loss": 1.5695,
+      "objective": 1.5490193367004395,
+      "ranking_idealized": 0.574999988079071,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 1.5490193367004395,
+      "step": 125
+    },
+    {
+      "dpo_loss": 0.9349392056465149,
+      "epoch": 0.7368918280585735,
+      "grad_norm": 70.71836455513233,
+      "learning_rate": 4.965385884295467e-06,
+      "logits": -1.109536051750183,
+      "logps": -70.39344024658203,
+      "loss": 1.5584,
+      "objective": 1.6358364820480347,
+      "ranking_idealized": 0.4833333194255829,
+      "ranking_idealized_expo": 0.44999998807907104,
+      "ranking_simple": 0.48750001192092896,
+      "regularize": 1.6358364820480347,
+      "step": 130
+    },
+    {
+      "dpo_loss": 0.9241760969161987,
+      "epoch": 0.7652338214454416,
+      "grad_norm": 71.12542306553239,
+      "learning_rate": 4.956679162840646e-06,
+      "logits": -1.092426061630249,
+      "logps": -73.43895721435547,
+      "loss": 1.5904,
+      "objective": 1.5671554803848267,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 1.5671554803848267,
+      "step": 135
+    },
+    {
+      "dpo_loss": 0.8581911325454712,
+      "epoch": 0.7935758148323099,
+      "grad_norm": 74.29158643708146,
+      "learning_rate": 4.947006115536947e-06,
+      "logits": -0.9535221457481384,
+      "logps": -74.29177856445312,
+      "loss": 1.5839,
+      "objective": 1.4818050861358643,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 1.4818050861358643,
+      "step": 140
+    },
+    {
+      "dpo_loss": 0.9467170238494873,
+      "epoch": 0.821917808219178,
+      "grad_norm": 69.83033897795461,
+      "learning_rate": 4.9363705472424825e-06,
+      "logits": -0.9065474271774292,
+      "logps": -75.74617004394531,
+      "loss": 1.5265,
+      "objective": 1.5216258764266968,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.4833333194255829,
+      "regularize": 1.5216258764266968,
+      "step": 145
+    },
+    {
+      "dpo_loss": 0.8972322940826416,
+      "epoch": 0.8502598016060463,
+      "grad_norm": 70.00491587575249,
+      "learning_rate": 4.924776641419513e-06,
+      "logits": -0.8950843811035156,
+      "logps": -74.2447509765625,
+      "loss": 1.5063,
+      "objective": 1.563843846321106,
+      "ranking_idealized": 0.5541666746139526,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5791666507720947,
+      "regularize": 1.563843846321106,
+      "step": 150
+    },
+    {
+      "epoch": 0.8502598016060463,
+      "eval_dpo_loss": 1.1387892961502075,
+      "eval_logits": -0.9763504266738892,
+      "eval_logps": -79.4244613647461,
+      "eval_loss": 1.840279221534729,
+      "eval_objective": 1.8307207822799683,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5154958963394165,
+      "eval_regularize": 1.8307207822799683,
+      "eval_runtime": 259.3897,
+      "eval_samples_per_second": 22.322,
+      "eval_steps_per_second": 0.933,
+      "step": 150
+    },
+    {
+      "dpo_loss": 0.8782619833946228,
+      "epoch": 0.8786017949929145,
+      "grad_norm": 69.80918589105492,
+      "learning_rate": 4.9122289584888926e-06,
+      "logits": -0.9591015577316284,
+      "logps": -72.81404876708984,
+      "loss": 1.5351,
+      "objective": 1.4583401679992676,
+      "ranking_idealized": 0.5541666746139526,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 1.4583401679992676,
+      "step": 155
+    },
+    {
+      "dpo_loss": 0.9660459756851196,
+      "epoch": 0.9069437883797827,
+      "grad_norm": 66.22318117843135,
+      "learning_rate": 4.8987324340362445e-06,
+      "logits": -0.9040888547897339,
+      "logps": -73.70983123779297,
+      "loss": 1.5597,
+      "objective": 1.562262773513794,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 1.562262773513794,
+      "step": 160
+    },
+    {
+      "dpo_loss": 1.0612136125564575,
+      "epoch": 0.9352857817666509,
+      "grad_norm": 70.50798575492288,
+      "learning_rate": 4.884292376870567e-06,
+      "logits": -0.8070274591445923,
+      "logps": -75.22913360595703,
+      "loss": 1.5216,
+      "objective": 1.6139070987701416,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 1.6139070987701416,
+      "step": 165
+    },
+    {
+      "dpo_loss": 0.8528857827186584,
+      "epoch": 0.9636277751535192,
+      "grad_norm": 70.82614721912003,
+      "learning_rate": 4.868914466936038e-06,
+      "logits": -0.6839962601661682,
+      "logps": -76.570068359375,
+      "loss": 1.483,
+      "objective": 1.5318470001220703,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 1.5318470001220703,
+      "step": 170
+    },
+    {
+      "dpo_loss": 0.8230343461036682,
+      "epoch": 0.9919697685403873,
+      "grad_norm": 73.28269267652773,
+      "learning_rate": 4.8526047530778175e-06,
+      "logits": -0.6581661105155945,
+      "logps": -75.77076721191406,
+      "loss": 1.4604,
+      "objective": 1.486951470375061,
+      "ranking_idealized": 0.5958333611488342,
+      "ranking_idealized_expo": 0.574999988079071,
+      "ranking_simple": 0.5958333611488342,
+      "regularize": 1.486951470375061,
+      "step": 175
+    },
+    {
+      "dpo_loss": 0.9295377135276794,
+      "epoch": 1.0203117619272555,
+      "grad_norm": 68.13139272327797,
+      "learning_rate": 4.835369650662767e-06,
+      "logits": -0.7830471396446228,
+      "logps": -75.41452026367188,
+      "loss": 1.4384,
+      "objective": 1.5701080560684204,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5541666746139526,
+      "regularize": 1.5701080560684204,
+      "step": 180
+    },
+    {
+      "dpo_loss": 0.9832867980003357,
+      "epoch": 1.0486537553141237,
+      "grad_norm": 68.27487274290651,
+      "learning_rate": 4.817215939055984e-06,
+      "logits": -0.7766852974891663,
+      "logps": -75.4141845703125,
+      "loss": 1.4597,
+      "objective": 1.5295368432998657,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.5,
+      "regularize": 1.5295368432998657,
+      "step": 185
+    },
+    {
+      "dpo_loss": 0.9126808047294617,
+      "epoch": 1.076995748700992,
+      "grad_norm": 72.02486344760798,
+      "learning_rate": 4.798150758954164e-06,
+      "logits": -0.7806929349899292,
+      "logps": -76.5052490234375,
+      "loss": 1.4245,
+      "objective": 1.6002991199493408,
+      "ranking_idealized": 0.5833333134651184,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 1.6002991199493408,
+      "step": 190
+    },
+    {
+      "dpo_loss": 0.876176655292511,
+      "epoch": 1.10533774208786,
+      "grad_norm": 72.39146825694654,
+      "learning_rate": 4.778181609576832e-06,
+      "logits": -0.742654025554657,
+      "logps": -75.56478881835938,
+      "loss": 1.4047,
+      "objective": 1.4097263813018799,
+      "ranking_idealized": 0.574999988079071,
+      "ranking_idealized_expo": 0.5666666626930237,
+      "ranking_simple": 0.5791666507720947,
+      "regularize": 1.4097263813018799,
+      "step": 195
+    },
+    {
+      "dpo_loss": 0.8269080519676208,
+      "epoch": 1.1336797354747283,
+      "grad_norm": 65.92725380936474,
+      "learning_rate": 4.757316345716554e-06,
+      "logits": -0.7813295125961304,
+      "logps": -73.13416290283203,
+      "loss": 1.3427,
+      "objective": 1.2968580722808838,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5083333253860474,
+      "regularize": 1.2968580722808838,
+      "step": 200
+    },
+    {
+      "epoch": 1.1336797354747283,
+      "eval_dpo_loss": 1.19429349899292,
+      "eval_logits": -0.844602108001709,
+      "eval_logps": -78.08975982666016,
+      "eval_loss": 1.9410725831985474,
+      "eval_objective": 1.904191493988037,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5123966932296753,
+      "eval_regularize": 1.904191493988037,
+      "eval_runtime": 259.3374,
+      "eval_samples_per_second": 22.326,
+      "eval_steps_per_second": 0.933,
+      "step": 200
+    },
+    {
+      "dpo_loss": 0.9577538371086121,
+      "epoch": 1.1620217288615966,
+      "grad_norm": 65.93331247907419,
+      "learning_rate": 4.735563174649278e-06,
+      "logits": -0.8114036321640015,
+      "logps": -74.05875396728516,
+      "loss": 1.3942,
+      "objective": 1.4247114658355713,
+      "ranking_idealized": 0.5,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.48750001192092896,
+      "regularize": 1.4247114658355713,
+      "step": 205
+    },
+    {
+      "dpo_loss": 0.7938932180404663,
+      "epoch": 1.1903637222484649,
+      "grad_norm": 71.43014040389335,
+      "learning_rate": 4.7129306529060415e-06,
+      "logits": -0.883682131767273,
+      "logps": -74.72289276123047,
+      "loss": 1.397,
+      "objective": 1.3820544481277466,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 1.3820544481277466,
+      "step": 210
+    },
+    {
+      "dpo_loss": 0.8905848860740662,
+      "epoch": 1.2187057156353331,
+      "grad_norm": 73.75682155225577,
+      "learning_rate": 4.68942768290728e-06,
+      "logits": -0.8337818384170532,
+      "logps": -75.42349243164062,
+      "loss": 1.346,
+      "objective": 1.3334628343582153,
+      "ranking_idealized": 0.49166667461395264,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.5,
+      "regularize": 1.3334628343582153,
+      "step": 215
+    },
+    {
+      "dpo_loss": 0.7579005360603333,
+      "epoch": 1.2470477090222012,
+      "grad_norm": 67.10552679086015,
+      "learning_rate": 4.665063509461098e-06,
+      "logits": -0.6136857867240906,
+      "logps": -76.30582427978516,
+      "loss": 1.3537,
+      "objective": 1.1673569679260254,
+      "ranking_idealized": 0.6083333492279053,
+      "ranking_idealized_expo": 0.6000000238418579,
+      "ranking_simple": 0.5916666388511658,
+      "regularize": 1.1673569679260254,
+      "step": 220
+    },
+    {
+      "dpo_loss": 0.8865377306938171,
+      "epoch": 1.2753897024090695,
+      "grad_norm": 61.86276513251386,
+      "learning_rate": 4.639847716126855e-06,
+      "logits": -0.712106466293335,
+      "logps": -77.76806640625,
+      "loss": 1.2903,
+      "objective": 1.3318673372268677,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 1.3318673372268677,
+      "step": 225
+    },
+    {
+      "dpo_loss": 0.7389308214187622,
+      "epoch": 1.3037316957959377,
+      "grad_norm": 64.8981585658753,
+      "learning_rate": 4.613790221445511e-06,
+      "logits": -0.7096338868141174,
+      "logps": -77.84996032714844,
+      "loss": 1.2526,
+      "objective": 1.1143450736999512,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 1.1143450736999512,
+      "step": 230
+    },
+    {
+      "dpo_loss": 0.7792016267776489,
+      "epoch": 1.3320736891828058,
+      "grad_norm": 70.5473568363482,
+      "learning_rate": 4.586901275038201e-06,
+      "logits": -0.8352426886558533,
+      "logps": -74.80459594726562,
+      "loss": 1.3276,
+      "objective": 1.2759090662002563,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 1.2759090662002563,
+      "step": 235
+    },
+    {
+      "dpo_loss": 0.7532822489738464,
+      "epoch": 1.360415682569674,
+      "grad_norm": 66.15538794121927,
+      "learning_rate": 4.559191453574582e-06,
+      "logits": -0.7979737520217896,
+      "logps": -75.91637420654297,
+      "loss": 1.2644,
+      "objective": 1.1841062307357788,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.5083333253860474,
+      "regularize": 1.1841062307357788,
+      "step": 240
+    },
+    {
+      "dpo_loss": 0.8654258847236633,
+      "epoch": 1.3887576759565423,
+      "grad_norm": 60.664093680465626,
+      "learning_rate": 4.530671656612544e-06,
+      "logits": -0.7953319549560547,
+      "logps": -75.70112609863281,
+      "loss": 1.2143,
+      "objective": 1.248257040977478,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 1.248257040977478,
+      "step": 245
+    },
+    {
+      "dpo_loss": 0.7265617847442627,
+      "epoch": 1.4170996693434104,
+      "grad_norm": 59.62226844787694,
+      "learning_rate": 4.501353102310901e-06,
+      "logits": -0.7469329833984375,
+      "logps": -75.32097625732422,
+      "loss": 1.2385,
+      "objective": 1.2008994817733765,
+      "ranking_idealized": 0.48750001192092896,
+      "ranking_idealized_expo": 0.47083333134651184,
+      "ranking_simple": 0.4833333194255829,
+      "regularize": 1.2008994817733765,
+      "step": 250
+    },
+    {
+      "epoch": 1.4170996693434104,
+      "eval_dpo_loss": 1.2812141180038452,
+      "eval_logits": -0.8252052664756775,
+      "eval_logps": -81.07825469970703,
+      "eval_loss": 2.100358247756958,
+      "eval_objective": 2.0779922008514404,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5072314143180847,
+      "eval_regularize": 2.0779922008514404,
+      "eval_runtime": 259.328,
+      "eval_samples_per_second": 22.327,
+      "eval_steps_per_second": 0.933,
+      "step": 250
+    },
+    {
+      "dpo_loss": 0.73033607006073,
+      "epoch": 1.4454416627302786,
+      "grad_norm": 62.42895757690262,
+      "learning_rate": 4.4712473230167775e-06,
+      "logits": -0.6974473595619202,
+      "logps": -77.66381072998047,
+      "loss": 1.2105,
+      "objective": 1.1618343591690063,
+      "ranking_idealized": 0.5708333253860474,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.6041666865348816,
+      "regularize": 1.1618343591690063,
+      "step": 255
+    },
+    {
+      "dpo_loss": 0.8443369269371033,
+      "epoch": 1.473783656117147,
+      "grad_norm": 61.04515459515312,
+      "learning_rate": 4.440366160729393e-06,
+      "logits": -0.705303430557251,
+      "logps": -77.5326156616211,
+      "loss": 1.1973,
+      "objective": 1.1547825336456299,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 1.1547825336456299,
+      "step": 260
+    },
+    {
+      "dpo_loss": 0.7983213663101196,
+      "epoch": 1.5021256495040152,
+      "grad_norm": 63.36584736778421,
+      "learning_rate": 4.4087217624420595e-06,
+      "logits": -0.6919764280319214,
+      "logps": -75.56660461425781,
+      "loss": 1.1823,
+      "objective": 1.208877444267273,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 1.208877444267273,
+      "step": 265
+    },
+    {
+      "dpo_loss": 0.719446063041687,
+      "epoch": 1.5304676428908834,
+      "grad_norm": 63.46733089619852,
+      "learning_rate": 4.376326575364206e-06,
+      "logits": -0.6683894991874695,
+      "logps": -76.26860809326172,
+      "loss": 1.1785,
+      "objective": 1.1638675928115845,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 1.1638675928115845,
+      "step": 270
+    },
+    {
+      "dpo_loss": 0.7665132284164429,
+      "epoch": 1.5588096362777515,
+      "grad_norm": 58.99295412475345,
+      "learning_rate": 4.34319334202531e-06,
+      "logits": -0.6416487097740173,
+      "logps": -76.59992218017578,
+      "loss": 1.1881,
+      "objective": 1.1479988098144531,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 1.1479988098144531,
+      "step": 275
+    },
+    {
+      "dpo_loss": 0.7462816834449768,
+      "epoch": 1.5871516296646198,
+      "grad_norm": 60.75435946216164,
+      "learning_rate": 4.309335095262675e-06,
+      "logits": -0.601517915725708,
+      "logps": -74.82725524902344,
+      "loss": 1.145,
+      "objective": 1.1755616664886475,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 1.1755616664886475,
+      "step": 280
+    },
+    {
+      "dpo_loss": 0.6989460587501526,
+      "epoch": 1.615493623051488,
+      "grad_norm": 64.56412273044151,
+      "learning_rate": 4.274765153095008e-06,
+      "logits": -0.644619882106781,
+      "logps": -76.7342758178711,
+      "loss": 1.1688,
+      "objective": 1.1958999633789062,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 1.1958999633789062,
+      "step": 285
+    },
+    {
+      "dpo_loss": 0.751794695854187,
+      "epoch": 1.643835616438356,
+      "grad_norm": 65.15922239336996,
+      "learning_rate": 4.239497113483819e-06,
+      "logits": -0.6588698625564575,
+      "logps": -74.58145141601562,
+      "loss": 1.1147,
+      "objective": 0.9951308369636536,
+      "ranking_idealized": 0.5708333253860474,
+      "ranking_idealized_expo": 0.5625,
+      "ranking_simple": 0.5625,
+      "regularize": 0.9951308369636536,
+      "step": 290
+    },
+    {
+      "dpo_loss": 0.7910669445991516,
+      "epoch": 1.6721776098252243,
+      "grad_norm": 59.311015532755086,
+      "learning_rate": 4.203544848984729e-06,
+      "logits": -0.5501735210418701,
+      "logps": -73.76367950439453,
+      "loss": 1.0785,
+      "objective": 1.1607948541641235,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 1.1607948541641235,
+      "step": 295
+    },
+    {
+      "dpo_loss": 0.787829577922821,
+      "epoch": 1.7005196032120926,
+      "grad_norm": 59.72417531410974,
+      "learning_rate": 4.16692250129073e-06,
+      "logits": -0.5574866533279419,
+      "logps": -74.32286834716797,
+      "loss": 1.1013,
+      "objective": 1.11799156665802,
+      "ranking_idealized": 0.5916666388511658,
+      "ranking_idealized_expo": 0.574999988079071,
+      "ranking_simple": 0.550000011920929,
+      "regularize": 1.11799156665802,
+      "step": 300
+    },
+    {
+      "epoch": 1.7005196032120926,
+      "eval_dpo_loss": 1.3091331720352173,
+      "eval_logits": -0.6190080046653748,
+      "eval_logps": -78.51614379882812,
+      "eval_loss": 2.195436716079712,
+      "eval_objective": 2.2003257274627686,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5123966932296753,
+      "eval_regularize": 2.2003257274627686,
+      "eval_runtime": 259.0657,
+      "eval_samples_per_second": 22.35,
+      "eval_steps_per_second": 0.934,
+      "step": 300
+    },
+    {
+      "dpo_loss": 0.751851499080658,
+      "epoch": 1.7288615965989607,
+      "grad_norm": 63.853803550753945,
+      "learning_rate": 4.129644475669617e-06,
+      "logits": -0.5315951108932495,
+      "logps": -74.70060729980469,
+      "loss": 1.1126,
+      "objective": 1.1344873905181885,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 1.1344873905181885,
+      "step": 305
+    },
+    {
+      "dpo_loss": 0.7526522278785706,
+      "epoch": 1.7572035899858292,
+      "grad_norm": 62.541695397328844,
+      "learning_rate": 4.091725435297721e-06,
+      "logits": -0.6003395915031433,
+      "logps": -71.47956848144531,
+      "loss": 1.104,
+      "objective": 1.1590847969055176,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 1.1590847969055176,
+      "step": 310
+    },
+    {
+      "dpo_loss": 0.7785258889198303,
+      "epoch": 1.7855455833726972,
+      "grad_norm": 56.52492221503349,
+      "learning_rate": 4.053180295492203e-06,
+      "logits": -0.5509213209152222,
+      "logps": -72.33855438232422,
+      "loss": 1.086,
+      "objective": 1.1518981456756592,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5249999761581421,
+      "regularize": 1.1518981456756592,
+      "step": 315
+    },
+    {
+      "dpo_loss": 0.7460355758666992,
+      "epoch": 1.8138875767595655,
+      "grad_norm": 60.186149725562295,
+      "learning_rate": 4.014024217844167e-06,
+      "logits": -0.5665237903594971,
+      "logps": -73.6473159790039,
+      "loss": 1.0789,
+      "objective": 1.1513078212738037,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 1.1513078212738037,
+      "step": 320
+    },
+    {
+      "dpo_loss": 0.7017228603363037,
+      "epoch": 1.8422295701464337,
+      "grad_norm": 64.38612525774512,
+      "learning_rate": 3.974272604254906e-06,
+      "logits": -0.5879048109054565,
+      "logps": -75.27242279052734,
+      "loss": 1.0824,
+      "objective": 1.1118178367614746,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5666666626930237,
+      "regularize": 1.1118178367614746,
+      "step": 325
+    },
+    {
+      "dpo_loss": 0.6825158596038818,
+      "epoch": 1.8705715635333018,
+      "grad_norm": 59.75239189291987,
+      "learning_rate": 3.933941090877615e-06,
+      "logits": -0.4700223505496979,
+      "logps": -71.7901382446289,
+      "loss": 1.0596,
+      "objective": 0.9758342504501343,
+      "ranking_idealized": 0.48750001192092896,
+      "ranking_idealized_expo": 0.47083333134651184,
+      "ranking_simple": 0.4541666805744171,
+      "regularize": 0.9758342504501343,
+      "step": 330
+    },
+    {
+      "dpo_loss": 0.6629473567008972,
+      "epoch": 1.89891355692017,
+      "grad_norm": 61.304279089943705,
+      "learning_rate": 3.893045541966975e-06,
+      "logits": -0.617573618888855,
+      "logps": -72.22500610351562,
+      "loss": 1.0234,
+      "objective": 1.1313632726669312,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 1.1313632726669312,
+      "step": 335
+    },
+    {
+      "dpo_loss": 0.7187826633453369,
+      "epoch": 1.9272555503070383,
+      "grad_norm": 59.753785455065014,
+      "learning_rate": 3.8516020436389945e-06,
+      "logits": -0.560102105140686,
+      "logps": -75.12157440185547,
+      "loss": 1.0127,
+      "objective": 1.0483639240264893,
+      "ranking_idealized": 0.5541666746139526,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 1.0483639240264893,
+      "step": 340
+    },
+    {
+      "dpo_loss": 0.6934853792190552,
+      "epoch": 1.9555975436939064,
+      "grad_norm": 57.97084895323718,
+      "learning_rate": 3.8096268975436045e-06,
+      "logits": -0.5952755212783813,
+      "logps": -74.10826873779297,
+      "loss": 1.022,
+      "objective": 1.0359365940093994,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 1.0359365940093994,
+      "step": 345
+    },
+    {
+      "dpo_loss": 0.6595560908317566,
+      "epoch": 1.9839395370807746,
+      "grad_norm": 61.69795496268391,
+      "learning_rate": 3.767136614452458e-06,
+      "logits": -0.562609851360321,
+      "logps": -75.11538696289062,
+      "loss": 0.9795,
+      "objective": 0.9731999635696411,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.9731999635696411,
+      "step": 350
+    },
+    {
+      "epoch": 1.9839395370807746,
+      "eval_dpo_loss": 1.2865926027297974,
+      "eval_logits": -0.6907580494880676,
+      "eval_logps": -78.29139709472656,
+      "eval_loss": 2.2000670433044434,
+      "eval_objective": 2.1849544048309326,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5092975497245789,
+      "eval_regularize": 2.1849544048309326,
+      "eval_runtime": 259.7345,
+      "eval_samples_per_second": 22.292,
+      "eval_steps_per_second": 0.932,
+      "step": 350
+    },
+    {
+      "dpo_loss": 0.7099167108535767,
+      "epoch": 2.012281530467643,
+      "grad_norm": 61.48654412749636,
+      "learning_rate": 3.724147907764478e-06,
+      "logits": -0.5564183592796326,
+      "logps": -74.59834289550781,
+      "loss": 0.9585,
+      "objective": 1.041745901107788,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 1.041745901107788,
+      "step": 355
+    },
+    {
+      "dpo_loss": 0.7087345123291016,
+      "epoch": 2.040623523854511,
+      "grad_norm": 61.77503577871403,
+      "learning_rate": 3.6806776869317074e-06,
+      "logits": -0.5873940587043762,
+      "logps": -73.76538848876953,
+      "loss": 0.9533,
+      "objective": 0.9487842917442322,
+      "ranking_idealized": 0.5541666746139526,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.9487842917442322,
+      "step": 360
+    },
+    {
+      "dpo_loss": 0.6601606607437134,
+      "epoch": 2.0689655172413794,
+      "grad_norm": 62.194779314868725,
+      "learning_rate": 3.6367430508080283e-06,
+      "logits": -0.6485423445701599,
+      "logps": -74.84728240966797,
+      "loss": 0.9459,
+      "objective": 0.9501416683197021,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.9501416683197021,
+      "step": 365
+    },
+    {
+      "dpo_loss": 0.6689183712005615,
+      "epoch": 2.0973075106282475,
+      "grad_norm": 62.24375049929638,
+      "learning_rate": 3.5923612809233987e-06,
+      "logits": -0.6007247567176819,
+      "logps": -72.23168182373047,
+      "loss": 0.9236,
+      "objective": 0.9433914422988892,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.9433914422988892,
+      "step": 370
+    },
+    {
+      "dpo_loss": 0.6166276335716248,
+      "epoch": 2.1256495040151155,
+      "grad_norm": 55.9855513993403,
+      "learning_rate": 3.547549834686222e-06,
+      "logits": -0.6374623775482178,
+      "logps": -73.1789321899414,
+      "loss": 0.9218,
+      "objective": 0.9237673878669739,
+      "ranking_idealized": 0.625,
+      "ranking_idealized_expo": 0.5874999761581421,
+      "ranking_simple": 0.6083333492279053,
+      "regularize": 0.9237673878669739,
+      "step": 375
+    },
+    {
+      "dpo_loss": 0.7234205007553101,
+      "epoch": 2.153991497401984,
+      "grad_norm": 60.877496205710976,
+      "learning_rate": 3.5023263385165346e-06,
+      "logits": -0.5776531100273132,
+      "logps": -72.65286254882812,
+      "loss": 0.8853,
+      "objective": 0.9437097311019897,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5249999761581421,
+      "regularize": 0.9437097311019897,
+      "step": 380
+    },
+    {
+      "dpo_loss": 0.706203818321228,
+      "epoch": 2.182333490788852,
+      "grad_norm": 57.432586251491834,
+      "learning_rate": 3.4567085809127247e-06,
+      "logits": -0.5748823881149292,
+      "logps": -75.21902465820312,
+      "loss": 0.8699,
+      "objective": 0.9273825287818909,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.9273825287818909,
+      "step": 385
+    },
+    {
+      "dpo_loss": 0.6570317149162292,
+      "epoch": 2.21067548417572,
+      "grad_norm": 59.58556599473961,
+      "learning_rate": 3.410714505454486e-06,
+      "logits": -0.5105537176132202,
+      "logps": -73.1999282836914,
+      "loss": 0.8797,
+      "objective": 0.7775837779045105,
+      "ranking_idealized": 0.48750001192092896,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.4791666567325592,
+      "regularize": 0.7775837779045105,
+      "step": 390
+    },
+    {
+      "dpo_loss": 0.6566095352172852,
+      "epoch": 2.2390174775625886,
+      "grad_norm": 61.70716494378518,
+      "learning_rate": 3.364362203744777e-06,
+      "logits": -0.48478570580482483,
+      "logps": -75.13355255126953,
+      "loss": 0.9257,
+      "objective": 0.9057443737983704,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.9057443737983704,
+      "step": 395
+    },
+    {
+      "dpo_loss": 0.6266034245491028,
+      "epoch": 2.2673594709494567,
+      "grad_norm": 61.76261428542362,
+      "learning_rate": 3.3176699082935546e-06,
+      "logits": -0.5294731855392456,
+      "logps": -76.35556030273438,
+      "loss": 0.8853,
+      "objective": 0.8211384415626526,
+      "ranking_idealized": 0.6000000238418579,
+      "ranking_idealized_expo": 0.574999988079071,
+      "ranking_simple": 0.6000000238418579,
+      "regularize": 0.8211384415626526,
+      "step": 400
+    },
+    {
+      "epoch": 2.2673594709494567,
+      "eval_dpo_loss": 1.3223490715026855,
+      "eval_logits": -0.6215647459030151,
+      "eval_logps": -78.5732421875,
+      "eval_loss": 2.267859697341919,
+      "eval_objective": 2.261888027191162,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5134297609329224,
+      "eval_regularize": 2.261888027191162,
+      "eval_runtime": 258.7107,
+      "eval_samples_per_second": 22.38,
+      "eval_steps_per_second": 0.935,
+      "step": 400
+    },
+    {
+      "dpo_loss": 0.7414664626121521,
+      "epoch": 2.295701464336325,
+      "grad_norm": 59.90696960742683,
+      "learning_rate": 3.2706559853460818e-06,
+      "logits": -0.6069660186767578,
+      "logps": -74.20203399658203,
+      "loss": 0.8443,
+      "objective": 0.830698549747467,
+      "ranking_idealized": 0.5833333134651184,
+      "ranking_idealized_expo": 0.5583333373069763,
+      "ranking_simple": 0.5666666626930237,
+      "regularize": 0.830698549747467,
+      "step": 405
+    },
+    {
+      "dpo_loss": 0.6774530410766602,
+      "epoch": 2.324043457723193,
+      "grad_norm": 55.80310964403766,
+      "learning_rate": 3.2233389276586325e-06,
+      "logits": -0.524910032749176,
+      "logps": -73.77034759521484,
+      "loss": 0.8216,
+      "objective": 0.7954517006874084,
+      "ranking_idealized": 0.48750001192092896,
+      "ranking_idealized_expo": 0.4749999940395355,
+      "ranking_simple": 0.4833333194255829,
+      "regularize": 0.7954517006874084,
+      "step": 410
+    },
+    {
+      "dpo_loss": 0.6917945742607117,
+      "epoch": 2.3523854511100613,
+      "grad_norm": 58.97332543380404,
+      "learning_rate": 3.1757373472244324e-06,
+      "logits": -0.5027827024459839,
+      "logps": -74.41297912597656,
+      "loss": 0.8371,
+      "objective": 0.8500952124595642,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.550000011920929,
+      "regularize": 0.8500952124595642,
+      "step": 415
+    },
+    {
+      "dpo_loss": 0.7004589438438416,
+      "epoch": 2.3807274444969297,
+      "grad_norm": 58.55747179048871,
+      "learning_rate": 3.127869967952698e-06,
+      "logits": -0.42817217111587524,
+      "logps": -76.00125122070312,
+      "loss": 0.8196,
+      "objective": 0.8261451125144958,
+      "ranking_idealized": 0.4791666567325592,
+      "ranking_idealized_expo": 0.47083333134651184,
+      "ranking_simple": 0.4791666567325592,
+      "regularize": 0.8261451125144958,
+      "step": 420
+    },
+    {
+      "dpo_loss": 0.6052082777023315,
+      "epoch": 2.409069437883798,
+      "grad_norm": 64.29650245686375,
+      "learning_rate": 3.0797556183036582e-06,
+      "logits": -0.48321303725242615,
+      "logps": -74.37902069091797,
+      "loss": 0.8213,
+      "objective": 0.7835978865623474,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.7835978865623474,
+      "step": 425
+    },
+    {
+      "dpo_loss": 0.6209239959716797,
+      "epoch": 2.4374114312706663,
+      "grad_norm": 57.31240800051826,
+      "learning_rate": 3.0314132238824416e-06,
+      "logits": -0.5284911394119263,
+      "logps": -74.74346160888672,
+      "loss": 0.7639,
+      "objective": 0.7543562054634094,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5625,
+      "regularize": 0.7543562054634094,
+      "step": 430
+    },
+    {
+      "dpo_loss": 0.6278953552246094,
+      "epoch": 2.4657534246575343,
+      "grad_norm": 55.78371923658181,
+      "learning_rate": 2.9828617999947647e-06,
+      "logits": -0.6039460301399231,
+      "logps": -74.01110076904297,
+      "loss": 0.7742,
+      "objective": 0.7583914995193481,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.7583914995193481,
+      "step": 435
+    },
+    {
+      "dpo_loss": 0.6551163792610168,
+      "epoch": 2.4940954180444024,
+      "grad_norm": 57.20639534632825,
+      "learning_rate": 2.9341204441673267e-06,
+      "logits": -0.6006036996841431,
+      "logps": -73.72498321533203,
+      "loss": 0.8175,
+      "objective": 0.8128156661987305,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.8128156661987305,
+      "step": 440
+    },
+    {
+      "dpo_loss": 0.6569511890411377,
+      "epoch": 2.5224374114312704,
+      "grad_norm": 59.028297224317896,
+      "learning_rate": 2.8852083286358647e-06,
+      "logits": -0.6477051973342896,
+      "logps": -71.30754852294922,
+      "loss": 0.7389,
+      "objective": 0.6897608637809753,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.6897608637809753,
+      "step": 445
+    },
+    {
+      "dpo_loss": 0.6735912561416626,
+      "epoch": 2.550779404818139,
+      "grad_norm": 62.76694554498361,
+      "learning_rate": 2.8361446928038298e-06,
+      "logits": -0.6754371523857117,
+      "logps": -73.46631622314453,
+      "loss": 0.7605,
+      "objective": 0.8445902466773987,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 0.8445902466773987,
+      "step": 450
+    },
+    {
+      "epoch": 2.550779404818139,
+      "eval_dpo_loss": 1.3571884632110596,
+      "eval_logits": -0.6825547218322754,
+      "eval_logps": -78.28398895263672,
+      "eval_loss": 2.2654569149017334,
+      "eval_objective": 2.2744035720825195,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5144628286361694,
+      "eval_regularize": 2.2744035720825195,
+      "eval_runtime": 259.3083,
+      "eval_samples_per_second": 22.329,
+      "eval_steps_per_second": 0.933,
+      "step": 450
+    },
+    {
+      "dpo_loss": 0.6667983531951904,
+      "epoch": 2.579121398205007,
+      "grad_norm": 58.3447211977729,
+      "learning_rate": 2.7869488356746344e-06,
+      "logits": -0.6426534056663513,
+      "logps": -74.59416961669922,
+      "loss": 0.7591,
+      "objective": 0.768614649772644,
+      "ranking_idealized": 0.5166666507720947,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.768614649772644,
+      "step": 455
+    },
+    {
+      "dpo_loss": 0.5558815598487854,
+      "epoch": 2.6074633915918755,
+      "grad_norm": 65.11260237104646,
+      "learning_rate": 2.7376401082604563e-06,
+      "logits": -0.6525983810424805,
+      "logps": -74.7845230102539,
+      "loss": 0.7638,
+      "objective": 0.691643476486206,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.691643476486206,
+      "step": 460
+    },
+    {
+      "dpo_loss": 0.6411501169204712,
+      "epoch": 2.6358053849787435,
+      "grad_norm": 62.60981601751369,
+      "learning_rate": 2.6882379059705953e-06,
+      "logits": -0.5221896767616272,
+      "logps": -75.2436752319336,
+      "loss": 0.7359,
+      "objective": 0.7508728504180908,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.7508728504180908,
+      "step": 465
+    },
+    {
+      "dpo_loss": 0.6434755325317383,
+      "epoch": 2.6641473783656116,
+      "grad_norm": 56.58366056844157,
+      "learning_rate": 2.6387616609823506e-06,
+      "logits": -0.5284293293952942,
+      "logps": -74.4692153930664,
+      "loss": 0.7269,
+      "objective": 0.6894288659095764,
+      "ranking_idealized": 0.5791666507720947,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.6894288659095764,
+      "step": 470
+    },
+    {
+      "dpo_loss": 0.6124823093414307,
+      "epoch": 2.69248937175248,
+      "grad_norm": 63.898667131172786,
+      "learning_rate": 2.5892308345974517e-06,
+      "logits": -0.5336421728134155,
+      "logps": -74.76774597167969,
+      "loss": 0.7216,
+      "objective": 0.6816112995147705,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.6816112995147705,
+      "step": 475
+    },
+    {
+      "dpo_loss": 0.6480967998504639,
+      "epoch": 2.720831365139348,
+      "grad_norm": 70.14559307144371,
+      "learning_rate": 2.53966490958702e-06,
+      "logits": -0.6288893222808838,
+      "logps": -73.35899353027344,
+      "loss": 0.722,
+      "objective": 0.7448421716690063,
+      "ranking_idealized": 0.5249999761581421,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.7448421716690063,
+      "step": 480
+    },
+    {
+      "dpo_loss": 0.6456955075263977,
+      "epoch": 2.7491733585262166,
+      "grad_norm": 55.88623408443581,
+      "learning_rate": 2.490083382528097e-06,
+      "logits": -0.6045793294906616,
+      "logps": -75.4445571899414,
+      "loss": 0.7142,
+      "objective": 0.7276442050933838,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.7276442050933838,
+      "step": 485
+    },
+    {
+      "dpo_loss": 0.600917637348175,
+      "epoch": 2.7775153519130846,
+      "grad_norm": 59.05306899045766,
+      "learning_rate": 2.440505756134732e-06,
+      "logits": -0.5500164031982422,
+      "logps": -74.36378479003906,
+      "loss": 0.7053,
+      "objective": 0.6555976867675781,
+      "ranking_idealized": 0.5041666626930237,
+      "ranking_idealized_expo": 0.47083333134651184,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 0.6555976867675781,
+      "step": 490
+    },
+    {
+      "dpo_loss": 0.645307719707489,
+      "epoch": 2.8058573452999527,
+      "grad_norm": 57.07656552315778,
+      "learning_rate": 2.3909515315866606e-06,
+      "logits": -0.5600085854530334,
+      "logps": -73.18649291992188,
+      "loss": 0.673,
+      "objective": 0.7392704486846924,
+      "ranking_idealized": 0.48750001192092896,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 0.7392704486846924,
+      "step": 495
+    },
+    {
+      "dpo_loss": 0.5844512581825256,
+      "epoch": 2.8341993386868207,
+      "grad_norm": 58.17740843884364,
+      "learning_rate": 2.341440200858589e-06,
+      "logits": -0.588214099407196,
+      "logps": -73.16432189941406,
+      "loss": 0.6709,
+      "objective": 0.6501026153564453,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.6501026153564453,
+      "step": 500
+    },
+    {
+      "epoch": 2.8341993386868207,
+      "eval_dpo_loss": 1.3375275135040283,
+      "eval_logits": -0.648563027381897,
+      "eval_logps": -79.71849822998047,
+      "eval_loss": 2.2687840461730957,
+      "eval_objective": 2.2577877044677734,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5185950398445129,
+      "eval_regularize": 2.2577877044677734,
+      "eval_runtime": 258.6203,
+      "eval_samples_per_second": 22.388,
+      "eval_steps_per_second": 0.936,
+      "step": 500
+    },
+    {
+      "dpo_loss": 0.6095997095108032,
+      "epoch": 2.862541332073689,
+      "grad_norm": 61.323078307757875,
+      "learning_rate": 2.2919912390530945e-06,
+      "logits": -0.5695565938949585,
+      "logps": -74.67700958251953,
+      "loss": 0.6466,
+      "objective": 0.6186994910240173,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.6186994910240173,
+      "step": 505
+    },
+    {
+      "dpo_loss": 0.6002517938613892,
+      "epoch": 2.8908833254605573,
+      "grad_norm": 57.75553868657935,
+      "learning_rate": 2.242624096740164e-06,
+      "logits": -0.5447255373001099,
+      "logps": -75.35613250732422,
+      "loss": 0.6493,
+      "objective": 0.6379550099372864,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.6379550099372864,
+      "step": 510
+    },
+    {
+      "dpo_loss": 0.6183599233627319,
+      "epoch": 2.9192253188474258,
+      "grad_norm": 56.91540287998159,
+      "learning_rate": 2.193358192306384e-06,
+      "logits": -0.5950519442558289,
+      "logps": -75.38876342773438,
+      "loss": 0.6549,
+      "objective": 0.646551251411438,
+      "ranking_idealized": 0.49166667461395264,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.4958333373069763,
+      "regularize": 0.646551251411438,
+      "step": 515
+    },
+    {
+      "dpo_loss": 0.5939019918441772,
+      "epoch": 2.947567312234294,
+      "grad_norm": 56.28970465624792,
+      "learning_rate": 2.1442129043167877e-06,
+      "logits": -0.5761558413505554,
+      "logps": -77.31433868408203,
+      "loss": 0.6045,
+      "objective": 0.5895420908927917,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.5895420908927917,
+      "step": 520
+    },
+    {
+      "dpo_loss": 0.6051114201545715,
+      "epoch": 2.975909305621162,
+      "grad_norm": 59.57121178469057,
+      "learning_rate": 2.0952075638923656e-06,
+      "logits": -0.5915683507919312,
+      "logps": -75.8340835571289,
+      "loss": 0.611,
+      "objective": 0.5858420133590698,
+      "ranking_idealized": 0.5791666507720947,
+      "ranking_idealized_expo": 0.5666666626930237,
+      "ranking_simple": 0.5666666626930237,
+      "regularize": 0.5858420133590698,
+      "step": 525
+    },
+    {
+      "dpo_loss": 0.5843603610992432,
+      "epoch": 3.0042512990080303,
+      "grad_norm": 58.501519481588275,
+      "learning_rate": 2.046361447106244e-06,
+      "logits": -0.5682967901229858,
+      "logps": -75.45525360107422,
+      "loss": 0.6198,
+      "objective": 0.6126269698143005,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.6126269698143005,
+      "step": 530
+    },
+    {
+      "dpo_loss": 0.5678777098655701,
+      "epoch": 3.0325932923948984,
+      "grad_norm": 56.81311454823308,
+      "learning_rate": 1.997693767401503e-06,
+      "logits": -0.572821855545044,
+      "logps": -76.72408294677734,
+      "loss": 0.5527,
+      "objective": 0.5598254799842834,
+      "ranking_idealized": 0.5541666746139526,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5708333253860474,
+      "regularize": 0.5598254799842834,
+      "step": 535
+    },
+    {
+      "dpo_loss": 0.603760838508606,
+      "epoch": 3.0609352857817664,
+      "grad_norm": 55.24119887208726,
+      "learning_rate": 1.9492236680336486e-06,
+      "logits": -0.6185352802276611,
+      "logps": -75.44217681884766,
+      "loss": 0.5398,
+      "objective": 0.5248841047286987,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5625,
+      "regularize": 0.5248841047286987,
+      "step": 540
+    },
+    {
+      "dpo_loss": 0.6126823425292969,
+      "epoch": 3.089277279168635,
+      "grad_norm": 59.23770495109222,
+      "learning_rate": 1.9009702145406728e-06,
+      "logits": -0.6088476181030273,
+      "logps": -76.27928161621094,
+      "loss": 0.5381,
+      "objective": 0.521655797958374,
+      "ranking_idealized": 0.5,
+      "ranking_idealized_expo": 0.49166667461395264,
+      "ranking_simple": 0.5041666626930237,
+      "regularize": 0.521655797958374,
+      "step": 545
+    },
+    {
+      "dpo_loss": 0.5726143717765808,
+      "epoch": 3.117619272555503,
+      "grad_norm": 56.22075610887121,
+      "learning_rate": 1.852952387243698e-06,
+      "logits": -0.5207330584526062,
+      "logps": -77.1482162475586,
+      "loss": 0.5302,
+      "objective": 0.5327464938163757,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5541666746139526,
+      "regularize": 0.5327464938163757,
+      "step": 550
+    },
+    {
+      "epoch": 3.117619272555503,
+      "eval_dpo_loss": 1.3210206031799316,
+      "eval_logits": -0.6266895532608032,
+      "eval_logps": -80.1419448852539,
+      "eval_loss": 2.259814500808716,
+      "eval_objective": 2.2429771423339844,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.51962810754776,
+      "eval_regularize": 2.2429771423339844,
+      "eval_runtime": 259.8661,
+      "eval_samples_per_second": 22.281,
+      "eval_steps_per_second": 0.931,
+      "step": 550
+    },
+    {
+      "dpo_loss": 0.5455856919288635,
+      "epoch": 3.1459612659423715,
+      "grad_norm": 58.00758823689939,
+      "learning_rate": 1.8051890737811395e-06,
+      "logits": -0.4721320569515228,
+      "logps": -77.1658706665039,
+      "loss": 0.5207,
+      "objective": 0.5126071572303772,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.5625,
+      "regularize": 0.5126071572303772,
+      "step": 555
+    },
+    {
+      "dpo_loss": 0.5464572906494141,
+      "epoch": 3.1743032593292395,
+      "grad_norm": 55.95277148909765,
+      "learning_rate": 1.7576990616793139e-06,
+      "logits": -0.49959325790405273,
+      "logps": -74.85193634033203,
+      "loss": 0.529,
+      "objective": 0.4622223675251007,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 0.4622223675251007,
+      "step": 560
+    },
+    {
+      "dpo_loss": 0.5949917435646057,
+      "epoch": 3.2026452527161076,
+      "grad_norm": 59.42402517752121,
+      "learning_rate": 1.7105010309624381e-06,
+      "logits": -0.5134323835372925,
+      "logps": -76.86905670166016,
+      "loss": 0.5143,
+      "objective": 0.49525225162506104,
+      "ranking_idealized": 0.5625,
+      "ranking_idealized_expo": 0.5416666865348816,
+      "ranking_simple": 0.5666666626930237,
+      "regularize": 0.49525225162506104,
+      "step": 565
+    },
+    {
+      "dpo_loss": 0.5860133767127991,
+      "epoch": 3.230987246102976,
+      "grad_norm": 57.89045181223991,
+      "learning_rate": 1.6636135468049122e-06,
+      "logits": -0.5014190077781677,
+      "logps": -76.04814147949219,
+      "loss": 0.4959,
+      "objective": 0.48642733693122864,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 0.48642733693122864,
+      "step": 570
+    },
+    {
+      "dpo_loss": 0.5744526386260986,
+      "epoch": 3.259329239489844,
+      "grad_norm": 54.09506631184751,
+      "learning_rate": 1.617055052228768e-06,
+      "logits": -0.5578089356422424,
+      "logps": -76.63516998291016,
+      "loss": 0.5051,
+      "objective": 0.4754410982131958,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.4754410982131958,
+      "step": 575
+    },
+    {
+      "dpo_loss": 0.5690687298774719,
+      "epoch": 3.287671232876712,
+      "grad_norm": 57.935165692154584,
+      "learning_rate": 1.5708438608491816e-06,
+      "logits": -0.5680824518203735,
+      "logps": -77.54747772216797,
+      "loss": 0.5044,
+      "objective": 0.47244158387184143,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5541666746139526,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.47244158387184143,
+      "step": 580
+    },
+    {
+      "dpo_loss": 0.633466362953186,
+      "epoch": 3.3160132262635806,
+      "grad_norm": 61.57647838064842,
+      "learning_rate": 1.524998149670871e-06,
+      "logits": -0.5756654143333435,
+      "logps": -78.29401397705078,
+      "loss": 0.4977,
+      "objective": 0.5307816863059998,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.5307816863059998,
+      "step": 585
+    },
+    {
+      "dpo_loss": 0.594211220741272,
+      "epoch": 3.3443552196504487,
+      "grad_norm": 58.66653194688653,
+      "learning_rate": 1.479535951938243e-06,
+      "logits": -0.5960578918457031,
+      "logps": -77.94918060302734,
+      "loss": 0.4782,
+      "objective": 0.46626031398773193,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.46626031398773193,
+      "step": 590
+    },
+    {
+      "dpo_loss": 0.5661875605583191,
+      "epoch": 3.372697213037317,
+      "grad_norm": 55.75147905971709,
+      "learning_rate": 1.43447515004208e-06,
+      "logits": -0.514352560043335,
+      "logps": -77.09761047363281,
+      "loss": 0.474,
+      "objective": 0.5005635023117065,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.5005635023117065,
+      "step": 595
+    },
+    {
+      "dpo_loss": 0.5296740531921387,
+      "epoch": 3.4010392064241852,
+      "grad_norm": 56.950549116726854,
+      "learning_rate": 1.3898334684855647e-06,
+      "logits": -0.5301509499549866,
+      "logps": -76.18799591064453,
+      "loss": 0.4552,
+      "objective": 0.4339993894100189,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.512499988079071,
+      "regularize": 0.4339993894100189,
+      "step": 600
+    },
+    {
+      "epoch": 3.4010392064241852,
+      "eval_dpo_loss": 1.3297724723815918,
+      "eval_logits": -0.6006779670715332,
+      "eval_logps": -79.95822143554688,
+      "eval_loss": 2.2547333240509033,
+      "eval_objective": 2.237860679626465,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5123966932296753,
+      "eval_regularize": 2.237860679626465,
+      "eval_runtime": 258.6303,
+      "eval_samples_per_second": 22.387,
+      "eval_steps_per_second": 0.936,
+      "step": 600
+    },
+    {
+      "dpo_loss": 0.606982409954071,
+      "epoch": 3.4293811998110533,
+      "grad_norm": 55.47374374411107,
+      "learning_rate": 1.3456284669124159e-06,
+      "logits": -0.5279320478439331,
+      "logps": -78.00320434570312,
+      "loss": 0.4475,
+      "objective": 0.46218839287757874,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.46218839287757874,
+      "step": 605
+    },
+    {
+      "dpo_loss": 0.5908689498901367,
+      "epoch": 3.4577231931979218,
+      "grad_norm": 60.157606585793644,
+      "learning_rate": 1.301877533199859e-06,
+      "logits": -0.5327341556549072,
+      "logps": -76.2584457397461,
+      "loss": 0.455,
+      "objective": 0.4404201805591583,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.4404201805591583,
+      "step": 610
+    },
+    {
+      "dpo_loss": 0.5425374507904053,
+      "epoch": 3.48606518658479,
+      "grad_norm": 56.316614210866504,
+      "learning_rate": 1.2585978766191726e-06,
+      "logits": -0.5640905499458313,
+      "logps": -76.85858917236328,
+      "loss": 0.4328,
+      "objective": 0.4495522081851959,
+      "ranking_idealized": 0.4749999940395355,
+      "ranking_idealized_expo": 0.46666666865348816,
+      "ranking_simple": 0.48750001192092896,
+      "regularize": 0.4495522081851959,
+      "step": 615
+    },
+    {
+      "dpo_loss": 0.5412243008613586,
+      "epoch": 3.514407179971658,
+      "grad_norm": 59.449782514540445,
+      "learning_rate": 1.2158065210664848e-06,
+      "logits": -0.5073726773262024,
+      "logps": -75.8833999633789,
+      "loss": 0.45,
+      "objective": 0.40877532958984375,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.4958333373069763,
+      "regularize": 0.40877532958984375,
+      "step": 620
+    },
+    {
+      "dpo_loss": 0.5705503225326538,
+      "epoch": 3.5427491733585263,
+      "grad_norm": 60.75475260172388,
+      "learning_rate": 1.1735202983664803e-06,
+      "logits": -0.5309284925460815,
+      "logps": -75.0905990600586,
+      "loss": 0.4327,
+      "objective": 0.4199952185153961,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.4199952185153961,
+      "step": 625
+    },
+    {
+      "dpo_loss": 0.5579532384872437,
+      "epoch": 3.5710911667453944,
+      "grad_norm": 59.775794629962114,
+      "learning_rate": 1.1317558416516696e-06,
+      "logits": -0.5706030130386353,
+      "logps": -74.92536163330078,
+      "loss": 0.4206,
+      "objective": 0.3978765606880188,
+      "ranking_idealized": 0.5291666388511658,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5249999761581421,
+      "regularize": 0.3978765606880188,
+      "step": 630
+    },
+    {
+      "dpo_loss": 0.5702130794525146,
+      "epoch": 3.5994331601322624,
+      "grad_norm": 58.150159448234085,
+      "learning_rate": 1.0905295788197993e-06,
+      "logits": -0.5399581789970398,
+      "logps": -75.56917572021484,
+      "loss": 0.4106,
+      "objective": 0.4598848223686218,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 0.4598848223686218,
+      "step": 635
+    },
+    {
+      "dpo_loss": 0.5239140391349792,
+      "epoch": 3.627775153519131,
+      "grad_norm": 56.82700141030008,
+      "learning_rate": 1.049857726072005e-06,
+      "logits": -0.490993469953537,
+      "logps": -76.47193145751953,
+      "loss": 0.3946,
+      "objective": 0.41905397176742554,
+      "ranking_idealized": 0.4958333373069763,
+      "ranking_idealized_expo": 0.4625000059604645,
+      "ranking_simple": 0.5,
+      "regularize": 0.41905397176742554,
+      "step": 640
+    },
+    {
+      "dpo_loss": 0.576458215713501,
+      "epoch": 3.656117146905999,
+      "grad_norm": 56.89880418064234,
+      "learning_rate": 1.0097562815342215e-06,
+      "logits": -0.5056424140930176,
+      "logps": -75.477783203125,
+      "loss": 0.3926,
+      "objective": 0.38865312933921814,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.38865312933921814,
+      "step": 645
+    },
+    {
+      "dpo_loss": 0.5464863181114197,
+      "epoch": 3.6844591402928675,
+      "grad_norm": 56.60252182656229,
+      "learning_rate": 9.702410189643838e-07,
+      "logits": -0.5153852105140686,
+      "logps": -76.41727447509766,
+      "loss": 0.3981,
+      "objective": 0.380422979593277,
+      "ranking_idealized": 0.5166666507720947,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 0.380422979593277,
+      "step": 650
+    },
+    {
+      "epoch": 3.6844591402928675,
+      "eval_dpo_loss": 1.3237992525100708,
+      "eval_logits": -0.5994979739189148,
+      "eval_logps": -80.18803405761719,
+      "eval_loss": 2.254920482635498,
+      "eval_objective": 2.239741802215576,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5154958963394165,
+      "eval_regularize": 2.239741802215576,
+      "eval_runtime": 259.1878,
+      "eval_samples_per_second": 22.339,
+      "eval_steps_per_second": 0.934,
+      "step": 650
+    },
+    {
+      "dpo_loss": 0.5698133707046509,
+      "epoch": 3.7128011336797355,
+      "grad_norm": 60.818049397288426,
+      "learning_rate": 9.313274815478698e-07,
+      "logits": -0.5156189799308777,
+      "logps": -75.30026245117188,
+      "loss": 0.3817,
+      "objective": 0.3786256015300751,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.3786256015300751,
+      "step": 655
+    },
+    {
+      "dpo_loss": 0.5675181150436401,
+      "epoch": 3.7411431270666036,
+      "grad_norm": 57.75499051269253,
+      "learning_rate": 8.930309757836517e-07,
+      "logits": -0.5635392069816589,
+      "logps": -76.37955474853516,
+      "loss": 0.3793,
+      "objective": 0.37651321291923523,
+      "ranking_idealized": 0.5666666626930237,
+      "ranking_idealized_expo": 0.5541666746139526,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.37651321291923523,
+      "step": 660
+    },
+    {
+      "dpo_loss": 0.5627972483634949,
+      "epoch": 3.769485120453472,
+      "grad_norm": 55.789081356284925,
+      "learning_rate": 8.553665654635343e-07,
+      "logits": -0.5413954854011536,
+      "logps": -76.61347198486328,
+      "loss": 0.358,
+      "objective": 0.36223313212394714,
+      "ranking_idealized": 0.5208333134651184,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5208333134651184,
+      "regularize": 0.36223313212394714,
+      "step": 665
+    },
+    {
+      "dpo_loss": 0.565936267375946,
+      "epoch": 3.79782711384034,
+      "grad_norm": 56.9797576995613,
+      "learning_rate": 8.183490657468687e-07,
+      "logits": -0.5987848043441772,
+      "logps": -76.39659881591797,
+      "loss": 0.3404,
+      "objective": 0.33699342608451843,
+      "ranking_idealized": 0.6041666865348816,
+      "ranking_idealized_expo": 0.5916666388511658,
+      "ranking_simple": 0.6041666865348816,
+      "regularize": 0.33699342608451843,
+      "step": 670
+    },
+    {
+      "dpo_loss": 0.5550724267959595,
+      "epoch": 3.826169107227208,
+      "grad_norm": 55.12534400843312,
+      "learning_rate": 7.819930373330669e-07,
+      "logits": -0.583532989025116,
+      "logps": -75.4254150390625,
+      "loss": 0.337,
+      "objective": 0.33032530546188354,
+      "ranking_idealized": 0.5666666626930237,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.5791666507720947,
+      "regularize": 0.33032530546188354,
+      "step": 675
+    },
+    {
+      "dpo_loss": 0.5683528780937195,
+      "epoch": 3.8545111006140766,
+      "grad_norm": 58.24346477498415,
+      "learning_rate": 7.463127807341966e-07,
+      "logits": -0.5395128130912781,
+      "logps": -75.92244720458984,
+      "loss": 0.3391,
+      "objective": 0.3696479797363281,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5249999761581421,
+      "regularize": 0.3696479797363281,
+      "step": 680
+    },
+    {
+      "dpo_loss": 0.5345361232757568,
+      "epoch": 3.8828530940009447,
+      "grad_norm": 60.795090855194246,
+      "learning_rate": 7.113223306499336e-07,
+      "logits": -0.5642789006233215,
+      "logps": -75.37647247314453,
+      "loss": 0.3278,
+      "objective": 0.30459117889404297,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5166666507720947,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.30459117889404297,
+      "step": 685
+    },
+    {
+      "dpo_loss": 0.5224164128303528,
+      "epoch": 3.9111950873878127,
+      "grad_norm": 53.66311212284037,
+      "learning_rate": 6.770354504470575e-07,
+      "logits": -0.6012124419212341,
+      "logps": -75.34683227539062,
+      "loss": 0.3242,
+      "objective": 0.30577850341796875,
+      "ranking_idealized": 0.5625,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.5833333134651184,
+      "regularize": 0.30577850341796875,
+      "step": 690
+    },
+    {
+      "dpo_loss": 0.5453590750694275,
+      "epoch": 3.9395370807746812,
+      "grad_norm": 57.79953484046242,
+      "learning_rate": 6.434656267456843e-07,
+      "logits": -0.5451433658599854,
+      "logps": -76.42092895507812,
+      "loss": 0.3124,
+      "objective": 0.324345201253891,
+      "ranking_idealized": 0.4791666567325592,
+      "ranking_idealized_expo": 0.46666666865348816,
+      "ranking_simple": 0.4791666567325592,
+      "regularize": 0.324345201253891,
+      "step": 695
+    },
+    {
+      "dpo_loss": 0.561674952507019,
+      "epoch": 3.9678790741615493,
+      "grad_norm": 54.53867904105972,
+      "learning_rate": 6.106260641143547e-07,
+      "logits": -0.5508003830909729,
+      "logps": -76.36372375488281,
+      "loss": 0.3178,
+      "objective": 0.31011003255844116,
+      "ranking_idealized": 0.49166667461395264,
+      "ranking_idealized_expo": 0.4833333194255829,
+      "ranking_simple": 0.48750001192092896,
+      "regularize": 0.31011003255844116,
+      "step": 700
+    },
+    {
+      "epoch": 3.9678790741615493,
+      "eval_dpo_loss": 1.3332420587539673,
+      "eval_logits": -0.62151700258255,
+      "eval_logps": -80.45596313476562,
+      "eval_loss": 2.2615702152252197,
+      "eval_objective": 2.2539472579956055,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5134297609329224,
+      "eval_regularize": 2.2539472579956055,
+      "eval_runtime": 259.1185,
+      "eval_samples_per_second": 22.345,
+      "eval_steps_per_second": 0.934,
+      "step": 700
+    },
+    {
+      "dpo_loss": 0.5321690440177917,
+      "epoch": 3.9962210675484178,
+      "grad_norm": 59.452065695343926,
+      "learning_rate": 5.785296798760601e-07,
+      "logits": -0.5087547898292542,
+      "logps": -75.18541717529297,
+      "loss": 0.3225,
+      "objective": 0.30213478207588196,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5625,
+      "regularize": 0.30213478207588196,
+      "step": 705
+    },
+    {
+      "dpo_loss": 0.5166592001914978,
+      "epoch": 4.024563060935286,
+      "grad_norm": 65.96197683204998,
+      "learning_rate": 5.471890990272666e-07,
+      "logits": -0.5448976159095764,
+      "logps": -76.6087875366211,
+      "loss": 0.2527,
+      "objective": 0.2436264157295227,
+      "ranking_idealized": 0.48750001192092896,
+      "ranking_idealized_expo": 0.4791666567325592,
+      "ranking_simple": 0.49166667461395264,
+      "regularize": 0.2436264157295227,
+      "step": 710
+    },
+    {
+      "dpo_loss": 0.5354845523834229,
+      "epoch": 4.052905054322154,
+      "grad_norm": 56.07206249663241,
+      "learning_rate": 5.166166492719124e-07,
+      "logits": -0.5357650518417358,
+      "logps": -75.84851837158203,
+      "loss": 0.2618,
+      "objective": 0.2531677484512329,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.2531677484512329,
+      "step": 715
+    },
+    {
+      "dpo_loss": 0.5235564112663269,
+      "epoch": 4.081247047709022,
+      "grad_norm": 57.7958952485207,
+      "learning_rate": 4.868243561723535e-07,
+      "logits": -0.49407169222831726,
+      "logps": -78.26904296875,
+      "loss": 0.2451,
+      "objective": 0.25101524591445923,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.25101524591445923,
+      "step": 720
+    },
+    {
+      "dpo_loss": 0.5391948819160461,
+      "epoch": 4.109589041095891,
+      "grad_norm": 55.93504200270769,
+      "learning_rate": 4.57823938419153e-07,
+      "logits": -0.5219827890396118,
+      "logps": -75.88143157958984,
+      "loss": 0.2571,
+      "objective": 0.2235218584537506,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.2235218584537506,
+      "step": 725
+    },
+    {
+      "dpo_loss": 0.5403502583503723,
+      "epoch": 4.137931034482759,
+      "grad_norm": 56.99961878175316,
+      "learning_rate": 4.2962680322157335e-07,
+      "logits": -0.5911334753036499,
+      "logps": -76.09339904785156,
+      "loss": 0.2446,
+      "objective": 0.230418860912323,
+      "ranking_idealized": 0.550000011920929,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.550000011920929,
+      "regularize": 0.230418860912323,
+      "step": 730
+    },
+    {
+      "dpo_loss": 0.5401233434677124,
+      "epoch": 4.166273027869627,
+      "grad_norm": 59.493122058302546,
+      "learning_rate": 4.0224404182059443e-07,
+      "logits": -0.5024449229240417,
+      "logps": -77.3831558227539,
+      "loss": 0.2499,
+      "objective": 0.24507470428943634,
+      "ranking_idealized": 0.5541666746139526,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.24507470428943634,
+      "step": 735
+    },
+    {
+      "dpo_loss": 0.5141600966453552,
+      "epoch": 4.194615021256495,
+      "grad_norm": 56.30361364827208,
+      "learning_rate": 3.756864251262143e-07,
+      "logits": -0.5385380983352661,
+      "logps": -76.72105407714844,
+      "loss": 0.2374,
+      "objective": 0.24047289788722992,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5208333134651184,
+      "ranking_simple": 0.5333333611488342,
+      "regularize": 0.24047289788722992,
+      "step": 740
+    },
+    {
+      "dpo_loss": 0.5445539951324463,
+      "epoch": 4.222957014643363,
+      "grad_norm": 56.95695963320432,
+      "learning_rate": 3.499643994807486e-07,
+      "logits": -0.5771783590316772,
+      "logps": -74.20252990722656,
+      "loss": 0.2476,
+      "objective": 0.24541395902633667,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.4958333373069763,
+      "regularize": 0.24541395902633667,
+      "step": 745
+    },
+    {
+      "dpo_loss": 0.5280672907829285,
+      "epoch": 4.251299008030231,
+      "grad_norm": 54.37624429775478,
+      "learning_rate": 3.250880825498026e-07,
+      "logits": -0.609397828578949,
+      "logps": -75.99716186523438,
+      "loss": 0.2213,
+      "objective": 0.2197273075580597,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.2197273075580597,
+      "step": 750
+    },
+    {
+      "epoch": 4.251299008030231,
+      "eval_dpo_loss": 1.3297063112258911,
+      "eval_logits": -0.6153666973114014,
+      "eval_logps": -80.15010833740234,
+      "eval_loss": 2.26202392578125,
+      "eval_objective": 2.2499067783355713,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5134297609329224,
+      "eval_regularize": 2.2499067783355713,
+      "eval_runtime": 258.6337,
+      "eval_samples_per_second": 22.387,
+      "eval_steps_per_second": 0.936,
+      "step": 750
+    },
+    {
+      "dpo_loss": 0.5372040271759033,
+      "epoch": 4.2796410014171,
+      "grad_norm": 60.72195132083163,
+      "learning_rate": 3.0106725934252095e-07,
+      "logits": -0.5743114948272705,
+      "logps": -75.755859375,
+      "loss": 0.2404,
+      "objective": 0.22062353789806366,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5333333611488342,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.22062353789806366,
+      "step": 755
+    },
+    {
+      "dpo_loss": 0.541923463344574,
+      "epoch": 4.307982994803968,
+      "grad_norm": 59.60019088112453,
+      "learning_rate": 2.779113783626916e-07,
+      "logits": -0.5603171586990356,
+      "logps": -75.97834777832031,
+      "loss": 0.2213,
+      "objective": 0.20643459260463715,
+      "ranking_idealized": 0.5791666507720947,
+      "ranking_idealized_expo": 0.5625,
+      "ranking_simple": 0.574999988079071,
+      "regularize": 0.20643459260463715,
+      "step": 760
+    },
+    {
+      "dpo_loss": 0.519105851650238,
+      "epoch": 4.336324988190836,
+      "grad_norm": 57.13784414910619,
+      "learning_rate": 2.5562954789221164e-07,
+      "logits": -0.5705324411392212,
+      "logps": -76.40815734863281,
+      "loss": 0.2268,
+      "objective": 0.23255951702594757,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5249999761581421,
+      "ranking_simple": 0.5458333492279053,
+      "regularize": 0.23255951702594757,
+      "step": 765
+    },
+    {
+      "dpo_loss": 0.5270203351974487,
+      "epoch": 4.364666981577704,
+      "grad_norm": 57.88827391411817,
+      "learning_rate": 2.3423053240837518e-07,
+      "logits": -0.5453040599822998,
+      "logps": -75.09925842285156,
+      "loss": 0.2271,
+      "objective": 0.1950923055410385,
+      "ranking_idealized": 0.5416666865348816,
+      "ranking_idealized_expo": 0.5041666626930237,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.1950923055410385,
+      "step": 770
+    },
+    {
+      "dpo_loss": 0.5309551954269409,
+      "epoch": 4.393008974964572,
+      "grad_norm": 53.8846273259703,
+      "learning_rate": 2.137227491364016e-07,
+      "logits": -0.5447086691856384,
+      "logps": -76.1126480102539,
+      "loss": 0.2246,
+      "objective": 0.2088509202003479,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5249999761581421,
+      "regularize": 0.2088509202003479,
+      "step": 775
+    },
+    {
+      "dpo_loss": 0.5339339971542358,
+      "epoch": 4.42135096835144,
+      "grad_norm": 59.22579729731716,
+      "learning_rate": 1.941142647385469e-07,
+      "logits": -0.5564789175987244,
+      "logps": -75.06421661376953,
+      "loss": 0.2149,
+      "objective": 0.21485432982444763,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.5083333253860474,
+      "regularize": 0.21485432982444763,
+      "step": 780
+    },
+    {
+      "dpo_loss": 0.5728757381439209,
+      "epoch": 4.449692961738309,
+      "grad_norm": 58.36366674226128,
+      "learning_rate": 1.7541279214111277e-07,
+      "logits": -0.5958257913589478,
+      "logps": -75.01850891113281,
+      "loss": 0.2181,
+      "objective": 0.25161251425743103,
+      "ranking_idealized": 0.5458333492279053,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.25161251425743103,
+      "step": 785
+    },
+    {
+      "dpo_loss": 0.5186070203781128,
+      "epoch": 4.478034955125177,
+      "grad_norm": 54.916993664319754,
+      "learning_rate": 1.5762568750059604e-07,
+      "logits": -0.5728685259819031,
+      "logps": -77.33419036865234,
+      "loss": 0.2061,
+      "objective": 0.18065284192562103,
+      "ranking_idealized": 0.5833333134651184,
+      "ranking_idealized_expo": 0.5583333373069763,
+      "ranking_simple": 0.5874999761581421,
+      "regularize": 0.18065284192562103,
+      "step": 790
+    },
+    {
+      "dpo_loss": 0.5318642854690552,
+      "epoch": 4.506376948512045,
+      "grad_norm": 58.53760269495991,
+      "learning_rate": 1.4075994731016895e-07,
+      "logits": -0.49769601225852966,
+      "logps": -77.13802337646484,
+      "loss": 0.2015,
+      "objective": 0.18465597927570343,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.18465597927570343,
+      "step": 795
+    },
+    {
+      "dpo_loss": 0.5291022658348083,
+      "epoch": 4.534718941898913,
+      "grad_norm": 56.73087705325052,
+      "learning_rate": 1.2482220564763669e-07,
+      "logits": -0.47620663046836853,
+      "logps": -76.15321350097656,
+      "loss": 0.2032,
+      "objective": 0.1981140822172165,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.1981140822172165,
+      "step": 800
+    },
+    {
+      "epoch": 4.534718941898913,
+      "eval_dpo_loss": 1.3294748067855835,
+      "eval_logits": -0.6175000667572021,
+      "eval_logps": -80.12406158447266,
+      "eval_loss": 2.2583107948303223,
+      "eval_objective": 2.2455341815948486,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5134297609329224,
+      "eval_regularize": 2.2455341815948486,
+      "eval_runtime": 259.1188,
+      "eval_samples_per_second": 22.345,
+      "eval_steps_per_second": 0.934,
+      "step": 800
+    },
+    {
+      "dpo_loss": 0.5472068786621094,
+      "epoch": 4.563060935285781,
+      "grad_norm": 55.12256671902985,
+      "learning_rate": 1.0981873156594381e-07,
+      "logits": -0.5433961153030396,
+      "logps": -75.67558288574219,
+      "loss": 0.2028,
+      "objective": 0.21665817499160767,
+      "ranking_idealized": 0.4833333194255829,
+      "ranking_idealized_expo": 0.4625000059604645,
+      "ranking_simple": 0.48750001192092896,
+      "regularize": 0.21665817499160767,
+      "step": 805
+    },
+    {
+      "dpo_loss": 0.5366577506065369,
+      "epoch": 4.59140292867265,
+      "grad_norm": 57.38834520577077,
+      "learning_rate": 9.575542662726756e-08,
+      "logits": -0.574076235294342,
+      "logps": -74.78955078125,
+      "loss": 0.1956,
+      "objective": 0.22629684209823608,
+      "ranking_idealized": 0.46666666865348816,
+      "ranking_idealized_expo": 0.4583333432674408,
+      "ranking_simple": 0.4749999940395355,
+      "regularize": 0.22629684209823608,
+      "step": 810
+    },
+    {
+      "dpo_loss": 0.5316675901412964,
+      "epoch": 4.619744922059518,
+      "grad_norm": 59.23667969355199,
+      "learning_rate": 8.26378225816582e-08,
+      "logits": -0.48441505432128906,
+      "logps": -76.50004577636719,
+      "loss": 0.1986,
+      "objective": 0.195883110165596,
+      "ranking_idealized": 0.6166666746139526,
+      "ranking_idealized_expo": 0.5874999761581421,
+      "ranking_simple": 0.6083333492279053,
+      "regularize": 0.195883110165596,
+      "step": 815
+    },
+    {
+      "dpo_loss": 0.5366373658180237,
+      "epoch": 4.648086915446386,
+      "grad_norm": 55.28284867900087,
+      "learning_rate": 7.047107919114588e-08,
+      "logits": -0.5500171184539795,
+      "logps": -75.87386322021484,
+      "loss": 0.2028,
+      "objective": 0.20712776482105255,
+      "ranking_idealized": 0.5625,
+      "ranking_idealized_expo": 0.5625,
+      "ranking_simple": 0.5666666626930237,
+      "regularize": 0.20712776482105255,
+      "step": 820
+    },
+    {
+      "dpo_loss": 0.5389170050621033,
+      "epoch": 4.6764289088332545,
+      "grad_norm": 57.21598882034636,
+      "learning_rate": 5.92599822001666e-08,
+      "logits": -0.5184782147407532,
+      "logps": -74.3449478149414,
+      "loss": 0.1976,
+      "objective": 0.18227988481521606,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.5,
+      "ranking_simple": 0.5083333253860474,
+      "regularize": 0.18227988481521606,
+      "step": 825
+    },
+    {
+      "dpo_loss": 0.5467627048492432,
+      "epoch": 4.7047709022201225,
+      "grad_norm": 54.82857089312232,
+      "learning_rate": 4.9008941453107527e-08,
+      "logits": -0.5783690214157104,
+      "logps": -76.44660186767578,
+      "loss": 0.2072,
+      "objective": 0.19597838819026947,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.512499988079071,
+      "ranking_simple": 0.5416666865348816,
+      "regularize": 0.19597838819026947,
+      "step": 830
+    },
+    {
+      "dpo_loss": 0.5296519994735718,
+      "epoch": 4.733112895606991,
+      "grad_norm": 54.828212438355145,
+      "learning_rate": 3.972198915970976e-08,
+      "logits": -0.5317445993423462,
+      "logps": -75.7918701171875,
+      "loss": 0.2013,
+      "objective": 0.20263001322746277,
+      "ranking_idealized": 0.512499988079071,
+      "ranking_idealized_expo": 0.48750001192092896,
+      "ranking_simple": 0.5083333253860474,
+      "regularize": 0.20263001322746277,
+      "step": 835
+    },
+    {
+      "dpo_loss": 0.5090625882148743,
+      "epoch": 4.7614548889938595,
+      "grad_norm": 55.3362837507438,
+      "learning_rate": 3.1402778309014284e-08,
+      "logits": -0.5363056063652039,
+      "logps": -76.1655044555664,
+      "loss": 0.1893,
+      "objective": 0.19915518164634705,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5541666746139526,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.19915518164634705,
+      "step": 840
+    },
+    {
+      "dpo_loss": 0.5551865100860596,
+      "epoch": 4.7897968823807275,
+      "grad_norm": 54.49437843853531,
+      "learning_rate": 2.4054581232470785e-08,
+      "logits": -0.541746199131012,
+      "logps": -76.23973083496094,
+      "loss": 0.1902,
+      "objective": 0.20770463347434998,
+      "ranking_idealized": 0.4791666567325592,
+      "ranking_idealized_expo": 0.4541666805744171,
+      "ranking_simple": 0.4791666567325592,
+      "regularize": 0.20770463347434998,
+      "step": 845
+    },
+    {
+      "dpo_loss": 0.5436845421791077,
+      "epoch": 4.818138875767596,
+      "grad_norm": 57.49186751897613,
+      "learning_rate": 1.768028831677926e-08,
+      "logits": -0.5542425513267517,
+      "logps": -74.9803237915039,
+      "loss": 0.1935,
+      "objective": 0.17371943593025208,
+      "ranking_idealized": 0.5333333611488342,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5375000238418579,
+      "regularize": 0.17371943593025208,
+      "step": 850
+    },
+    {
+      "epoch": 4.818138875767596,
+      "eval_dpo_loss": 1.3283706903457642,
+      "eval_logits": -0.6168581247329712,
+      "eval_logps": -80.06609344482422,
+      "eval_loss": 2.2561168670654297,
+      "eval_objective": 2.242354154586792,
+      "eval_ranking_idealized": 0.5247933864593506,
+      "eval_ranking_idealized_expo": 0.5092975497245789,
+      "eval_ranking_simple": 0.5134297609329224,
+      "eval_regularize": 2.242354154586792,
+      "eval_runtime": 259.267,
+      "eval_samples_per_second": 22.332,
+      "eval_steps_per_second": 0.933,
+      "step": 850
+    },
+    {
+      "dpo_loss": 0.5587651133537292,
+      "epoch": 4.846480869154464,
+      "grad_norm": 57.56815145820478,
+      "learning_rate": 1.2282406866966078e-08,
+      "logits": -0.5165177583694458,
+      "logps": -75.15120697021484,
+      "loss": 0.1923,
+      "objective": 0.2012535035610199,
+      "ranking_idealized": 0.5583333373069763,
+      "ranking_idealized_expo": 0.5375000238418579,
+      "ranking_simple": 0.550000011920929,
+      "regularize": 0.2012535035610199,
+      "step": 855
+    },
+    {
+      "dpo_loss": 0.5044924020767212,
+      "epoch": 4.874822862541333,
+      "grad_norm": 57.45096886360892,
+      "learning_rate": 7.863060120144316e-09,
+      "logits": -0.508831799030304,
+      "logps": -75.34513854980469,
+      "loss": 0.1944,
+      "objective": 0.2123396247625351,
+      "ranking_idealized": 0.5666666626930237,
+      "ranking_idealized_expo": 0.550000011920929,
+      "ranking_simple": 0.574999988079071,
+      "regularize": 0.2123396247625351,
+      "step": 860
+    },
+    {
+      "dpo_loss": 0.52206951379776,
+      "epoch": 4.903164855928201,
+      "grad_norm": 58.02043264781572,
+      "learning_rate": 4.423986410346526e-09,
+      "logits": -0.5500971078872681,
+      "logps": -74.25553131103516,
+      "loss": 0.1747,
+      "objective": 0.16172558069229126,
+      "ranking_idealized": 0.5083333253860474,
+      "ranking_idealized_expo": 0.4958333373069763,
+      "ranking_simple": 0.5166666507720947,
+      "regularize": 0.16172558069229126,
+      "step": 865
+    },
+    {
+      "dpo_loss": 0.5237457752227783,
+      "epoch": 4.931506849315069,
+      "grad_norm": 55.98430316965056,
+      "learning_rate": 1.9665384847583622e-09,
+      "logits": -0.5589514970779419,
+      "logps": -75.981689453125,
+      "loss": 0.1822,
+      "objective": 0.18714649975299835,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5083333253860474,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.18714649975299835,
+      "step": 870
+    },
+    {
+      "dpo_loss": 0.5439503192901611,
+      "epoch": 4.959848842701937,
+      "grad_norm": 55.03768486905713,
+      "learning_rate": 4.916829716183901e-10,
+      "logits": -0.5287134051322937,
+      "logps": -75.68090057373047,
+      "loss": 0.1955,
+      "objective": 0.20633025467395782,
+      "ranking_idealized": 0.574999988079071,
+      "ranking_idealized_expo": 0.5458333492279053,
+      "ranking_simple": 0.5583333373069763,
+      "regularize": 0.20633025467395782,
+      "step": 875
+    },
+    {
+      "dpo_loss": 0.5364298224449158,
+      "epoch": 4.988190836088805,
+      "grad_norm": 56.14268444647409,
+      "learning_rate": 0.0,
+      "logits": -0.5877522826194763,
+      "logps": -76.51420593261719,
+      "loss": 0.1905,
+      "objective": 0.20404843986034393,
+      "ranking_idealized": 0.5375000238418579,
+      "ranking_idealized_expo": 0.5291666388511658,
+      "ranking_simple": 0.5291666388511658,
+      "regularize": 0.20404843986034393,
+      "step": 880
+    },
+    {
+      "epoch": 4.988190836088805,
+      "step": 880,
+      "total_flos": 0.0,
+      "train_loss": 0.7624400413849137,
+      "train_runtime": 35099.769,
+      "train_samples_per_second": 7.237,
+      "train_steps_per_second": 0.025
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 880,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}