{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.3615982643283312,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0036159826432833123,
      "grad_norm": 0.3235728144645691,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.043861389160156,
      "logits/rejected": -18.043861389160156,
      "logps/chosen": -0.6726381778717041,
      "logps/rejected": -0.6726381778717041,
      "loss": 6.9675,
      "nll_loss": 1.6725690364837646,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06726382672786713,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06726382672786713,
      "step": 10
    },
    {
      "epoch": 0.0072319652865666245,
      "grad_norm": 0.3509086072444916,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.013948440551758,
      "logits/rejected": -18.013948440551758,
      "logps/chosen": -0.6865767240524292,
      "logps/rejected": -0.6865767240524292,
      "loss": 6.7227,
      "nll_loss": 1.6113628149032593,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06865767389535904,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06865767389535904,
      "step": 20
    },
    {
      "epoch": 0.010847947929849937,
      "grad_norm": 0.38406211137771606,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.367061614990234,
      "logits/rejected": -18.367061614990234,
      "logps/chosen": -0.6837120056152344,
      "logps/rejected": -0.6837120056152344,
      "loss": 6.3986,
      "nll_loss": 1.5303384065628052,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06837119162082672,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06837119162082672,
      "step": 30
    },
    {
      "epoch": 0.014463930573133249,
      "grad_norm": 0.4101921319961548,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -17.99128532409668,
      "logits/rejected": -17.99128532409668,
      "logps/chosen": -0.6463029980659485,
      "logps/rejected": -0.6463029980659485,
      "loss": 7.0291,
      "nll_loss": 1.6879545450210571,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06463029980659485,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06463029980659485,
      "step": 40
    },
    {
      "epoch": 0.01807991321641656,
      "grad_norm": 0.444273978471756,
      "learning_rate": 5e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.08310890197754,
      "logits/rejected": -18.08310890197754,
      "logps/chosen": -0.6336182355880737,
      "logps/rejected": -0.6336182355880737,
      "loss": 6.5453,
      "nll_loss": 1.5670195817947388,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06336182355880737,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06336182355880737,
      "step": 50
    },
    {
      "epoch": 0.021695895859699874,
      "grad_norm": 0.67641681432724,
      "learning_rate": 6e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.269500732421875,
      "logits/rejected": -18.269500732421875,
      "logps/chosen": -0.6496673822402954,
      "logps/rejected": -0.6496673822402954,
      "loss": 6.733,
      "nll_loss": 1.6139262914657593,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06496674567461014,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06496674567461014,
      "step": 60
    },
    {
      "epoch": 0.025311878502983186,
      "grad_norm": 0.39636465907096863,
      "learning_rate": 7e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.115657806396484,
      "logits/rejected": -18.115657806396484,
      "logps/chosen": -0.6373459696769714,
      "logps/rejected": -0.6373459696769714,
      "loss": 6.8825,
      "nll_loss": 1.651307463645935,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06373460590839386,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06373460590839386,
      "step": 70
    },
    {
      "epoch": 0.028927861146266498,
      "grad_norm": 0.5290302634239197,
      "learning_rate": 8.000000000000001e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.138675689697266,
      "logits/rejected": -18.138675689697266,
      "logps/chosen": -0.6702518463134766,
      "logps/rejected": -0.6702518463134766,
      "loss": 6.4063,
      "nll_loss": 1.5322693586349487,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06702519208192825,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06702519208192825,
      "step": 80
    },
    {
      "epoch": 0.03254384378954981,
      "grad_norm": 0.2837754189968109,
      "learning_rate": 9e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.285091400146484,
      "logits/rejected": -18.285091400146484,
      "logps/chosen": -0.5695949196815491,
      "logps/rejected": -0.5695949196815491,
      "loss": 6.3599,
      "nll_loss": 1.520659327507019,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.056959498673677444,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.056959498673677444,
      "step": 90
    },
    {
      "epoch": 0.03615982643283312,
      "grad_norm": 0.38816723227500916,
      "learning_rate": 1e-05,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.38321304321289,
      "logits/rejected": -18.38321304321289,
      "logps/chosen": -0.6220490336418152,
      "logps/rejected": -0.6220490336418152,
      "loss": 6.3579,
      "nll_loss": 1.5201700925827026,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.062204908579587936,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.062204908579587936,
      "step": 100
    },
    {
      "epoch": 0.039775809076116435,
      "grad_norm": 0.485514760017395,
      "learning_rate": 9.88888888888889e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.24630355834961,
      "logits/rejected": -18.24630355834961,
      "logps/chosen": -0.58094322681427,
      "logps/rejected": -0.58094322681427,
      "loss": 6.8083,
      "nll_loss": 1.6327617168426514,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.058094322681427,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.058094322681427,
      "step": 110
    },
    {
      "epoch": 0.04339179171939975,
      "grad_norm": 0.7200180292129517,
      "learning_rate": 9.777777777777779e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.222978591918945,
      "logits/rejected": -18.222978591918945,
      "logps/chosen": -0.6140703558921814,
      "logps/rejected": -0.6140703558921814,
      "loss": 7.0165,
      "nll_loss": 1.6848167181015015,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06140704080462456,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06140704080462456,
      "step": 120
    },
    {
      "epoch": 0.04700777436268306,
      "grad_norm": 0.5130624175071716,
      "learning_rate": 9.666666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -17.98419189453125,
      "logits/rejected": -17.98419189453125,
      "logps/chosen": -0.6171376705169678,
      "logps/rejected": -0.6171376705169678,
      "loss": 6.9743,
      "nll_loss": 1.6742585897445679,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.06171376630663872,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.06171376630663872,
      "step": 130
    },
    {
      "epoch": 0.05062375700596637,
      "grad_norm": 0.6743359565734863,
      "learning_rate": 9.555555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.43941879272461,
      "logits/rejected": -18.43941879272461,
      "logps/chosen": -0.5736243724822998,
      "logps/rejected": -0.5736243724822998,
      "loss": 6.4179,
      "nll_loss": 1.5351518392562866,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.057362429797649384,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.057362429797649384,
      "step": 140
    },
    {
      "epoch": 0.054239739649249684,
      "grad_norm": 0.7007283568382263,
      "learning_rate": 9.444444444444445e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.523662567138672,
      "logits/rejected": -18.523662567138672,
      "logps/chosen": -0.5047087073326111,
      "logps/rejected": -0.5047087073326111,
      "loss": 6.4734,
      "nll_loss": 1.549041986465454,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.050470877438783646,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.050470877438783646,
      "step": 150
    },
    {
      "epoch": 0.057855722292532996,
      "grad_norm": 0.8055678009986877,
      "learning_rate": 9.333333333333334e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.57369041442871,
      "logits/rejected": -18.57369041442871,
      "logps/chosen": -0.48697715997695923,
      "logps/rejected": -0.48697715997695923,
      "loss": 6.1125,
      "nll_loss": 1.4588209390640259,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.04869771748781204,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.04869771748781204,
      "step": 160
    },
    {
      "epoch": 0.06147170493581631,
      "grad_norm": 0.5678962469100952,
      "learning_rate": 9.222222222222224e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.546266555786133,
      "logits/rejected": -18.546266555786133,
      "logps/chosen": -0.4713471531867981,
      "logps/rejected": -0.4713471531867981,
      "loss": 6.3675,
      "nll_loss": 1.5225670337677002,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.04713470861315727,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.04713470861315727,
      "step": 170
    },
    {
      "epoch": 0.06508768757909962,
      "grad_norm": 0.9227916598320007,
      "learning_rate": 9.111111111111112e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.552295684814453,
      "logits/rejected": -18.552295684814453,
      "logps/chosen": -0.5135122537612915,
      "logps/rejected": -0.5135122537612915,
      "loss": 6.3925,
      "nll_loss": 1.5288182497024536,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.05135122686624527,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.05135122686624527,
      "step": 180
    },
    {
      "epoch": 0.06870367022238293,
      "grad_norm": 1.3030140399932861,
      "learning_rate": 9e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.453067779541016,
      "logits/rejected": -18.453067779541016,
      "logps/chosen": -0.4881868362426758,
      "logps/rejected": -0.4881868362426758,
      "loss": 6.5908,
      "nll_loss": 1.5783830881118774,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.0488186851143837,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.0488186851143837,
      "step": 190
    },
    {
      "epoch": 0.07231965286566625,
      "grad_norm": 1.0002509355545044,
      "learning_rate": 8.888888888888888e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.477296829223633,
      "logits/rejected": -18.477296829223633,
      "logps/chosen": -0.46165475249290466,
      "logps/rejected": -0.46165475249290466,
      "loss": 6.6221,
      "nll_loss": 1.5862023830413818,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.046165481209754944,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.046165481209754944,
      "step": 200
    },
    {
      "epoch": 0.07593563550894955,
      "grad_norm": 0.7885683178901672,
      "learning_rate": 8.777777777777778e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.564895629882812,
      "logits/rejected": -18.564895629882812,
      "logps/chosen": -0.41595011949539185,
      "logps/rejected": -0.41595011949539185,
      "loss": 6.2789,
      "nll_loss": 1.500412940979004,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.04159501940011978,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.04159501940011978,
      "step": 210
    },
    {
      "epoch": 0.07955161815223287,
      "grad_norm": 0.6379426717758179,
      "learning_rate": 8.666666666666668e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.577289581298828,
      "logits/rejected": -18.577289581298828,
      "logps/chosen": -0.43426617980003357,
      "logps/rejected": -0.43426617980003357,
      "loss": 6.4009,
      "nll_loss": 1.5309035778045654,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.043426621705293655,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.043426621705293655,
      "step": 220
    },
    {
      "epoch": 0.08316760079551618,
      "grad_norm": 0.5890640020370483,
      "learning_rate": 8.555555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.817012786865234,
      "logits/rejected": -18.817012786865234,
      "logps/chosen": -0.38255172967910767,
      "logps/rejected": -0.38255172967910767,
      "loss": 6.021,
      "nll_loss": 1.4359278678894043,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.038255173712968826,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.038255173712968826,
      "step": 230
    },
    {
      "epoch": 0.0867835834387995,
      "grad_norm": 0.5675965547561646,
      "learning_rate": 8.444444444444446e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.88204002380371,
      "logits/rejected": -18.88204002380371,
      "logps/chosen": -0.357022225856781,
      "logps/rejected": -0.357022225856781,
      "loss": 6.3796,
      "nll_loss": 1.5255934000015259,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.03570222482085228,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.03570222482085228,
      "step": 240
    },
    {
      "epoch": 0.0903995660820828,
      "grad_norm": 0.284084677696228,
      "learning_rate": 8.344444444444445e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.659196853637695,
      "logits/rejected": -18.659196853637695,
      "logps/chosen": -0.35400137305259705,
      "logps/rejected": -0.35400137305259705,
      "loss": 6.1544,
      "nll_loss": 1.4692747592926025,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.03540014103055,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.03540014103055,
      "step": 250
    },
    {
      "epoch": 0.09401554872536612,
      "grad_norm": 0.6424693465232849,
      "learning_rate": 8.233333333333335e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.762996673583984,
      "logits/rejected": -18.762996673583984,
      "logps/chosen": -0.40258026123046875,
      "logps/rejected": -0.40258026123046875,
      "loss": 6.3857,
      "nll_loss": 1.5271098613739014,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.040258027613162994,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.040258027613162994,
      "step": 260
    },
    {
      "epoch": 0.09763153136864942,
      "grad_norm": 0.6682632565498352,
      "learning_rate": 8.122222222222223e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.719924926757812,
      "logits/rejected": -18.719924926757812,
      "logps/chosen": -0.3123939633369446,
      "logps/rejected": -0.3123939633369446,
      "loss": 5.7456,
      "nll_loss": 1.36708664894104,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.031239395961165428,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.031239395961165428,
      "step": 270
    },
    {
      "epoch": 0.10124751401193274,
      "grad_norm": 0.9669603109359741,
      "learning_rate": 8.011111111111113e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.799579620361328,
      "logits/rejected": -18.799579620361328,
      "logps/chosen": -0.3817462623119354,
      "logps/rejected": -0.3817462623119354,
      "loss": 6.2838,
      "nll_loss": 1.5016276836395264,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.03817462921142578,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.03817462921142578,
      "step": 280
    },
    {
      "epoch": 0.10486349665521605,
      "grad_norm": 0.7264003157615662,
      "learning_rate": 7.9e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.090185165405273,
      "logits/rejected": -19.090185165405273,
      "logps/chosen": -0.3294925093650818,
      "logps/rejected": -0.3294925093650818,
      "loss": 5.6699,
      "nll_loss": 1.3481695652008057,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.03294925019145012,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.03294925019145012,
      "step": 290
    },
    {
      "epoch": 0.10847947929849937,
      "grad_norm": 0.564611554145813,
      "learning_rate": 7.788888888888889e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.767759323120117,
      "logits/rejected": -18.767759323120117,
      "logps/chosen": -0.26777949929237366,
      "logps/rejected": -0.26777949929237366,
      "loss": 6.5104,
      "nll_loss": 1.5582915544509888,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.026777952909469604,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.026777952909469604,
      "step": 300
    },
    {
      "epoch": 0.11209546194178267,
      "grad_norm": 0.6952475905418396,
      "learning_rate": 7.677777777777778e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.896703720092773,
      "logits/rejected": -18.896703720092773,
      "logps/chosen": -0.24664482474327087,
      "logps/rejected": -0.24664482474327087,
      "loss": 5.9777,
      "nll_loss": 1.4251067638397217,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.02466448023915291,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.02466448023915291,
      "step": 310
    },
    {
      "epoch": 0.11571144458506599,
      "grad_norm": 0.7094094753265381,
      "learning_rate": 7.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.993297576904297,
      "logits/rejected": -18.993297576904297,
      "logps/chosen": -0.27139216661453247,
      "logps/rejected": -0.27139216661453247,
      "loss": 5.848,
      "nll_loss": 1.3926928043365479,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.027139216661453247,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.027139216661453247,
      "step": 320
    },
    {
      "epoch": 0.1193274272283493,
      "grad_norm": 0.8386672735214233,
      "learning_rate": 7.455555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.818639755249023,
      "logits/rejected": -18.818639755249023,
      "logps/chosen": -0.2643309533596039,
      "logps/rejected": -0.2643309533596039,
      "loss": 6.2247,
      "nll_loss": 1.4868563413619995,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.026433095335960388,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.026433095335960388,
      "step": 330
    },
    {
      "epoch": 0.12294340987163262,
      "grad_norm": 0.6337667107582092,
      "learning_rate": 7.344444444444445e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.07388687133789,
      "logits/rejected": -19.07388687133789,
      "logps/chosen": -0.23179857432842255,
      "logps/rejected": -0.23179857432842255,
      "loss": 5.7871,
      "nll_loss": 1.3774592876434326,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.023179858922958374,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.023179858922958374,
      "step": 340
    },
    {
      "epoch": 0.12655939251491594,
      "grad_norm": 1.056504726409912,
      "learning_rate": 7.233333333333334e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.8520450592041,
      "logits/rejected": -18.8520450592041,
      "logps/chosen": -0.24602051079273224,
      "logps/rejected": -0.24602051079273224,
      "loss": 5.8268,
      "nll_loss": 1.3873755931854248,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.024602051824331284,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.024602051824331284,
      "step": 350
    },
    {
      "epoch": 0.13017537515819924,
      "grad_norm": 0.4462037682533264,
      "learning_rate": 7.122222222222222e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.04658317565918,
      "logits/rejected": -19.04658317565918,
      "logps/chosen": -0.2291136234998703,
      "logps/rejected": -0.2291136234998703,
      "loss": 5.8142,
      "nll_loss": 1.384232521057129,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.02291136048734188,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.02291136048734188,
      "step": 360
    },
    {
      "epoch": 0.13379135780148255,
      "grad_norm": 0.5579663515090942,
      "learning_rate": 7.011111111111112e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -18.912765502929688,
      "logits/rejected": -18.912765502929688,
      "logps/chosen": -0.26528915762901306,
      "logps/rejected": -0.26528915762901306,
      "loss": 6.0316,
      "nll_loss": 1.4385900497436523,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.026528915390372276,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.026528915390372276,
      "step": 370
    },
    {
      "epoch": 0.13740734044476585,
      "grad_norm": 1.2433960437774658,
      "learning_rate": 6.9e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.217700958251953,
      "logits/rejected": -19.217700958251953,
      "logps/chosen": -0.21606405079364777,
      "logps/rejected": -0.21606405079364777,
      "loss": 5.7944,
      "nll_loss": 1.3792803287506104,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.021606406196951866,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.021606406196951866,
      "step": 380
    },
    {
      "epoch": 0.14102332308804919,
      "grad_norm": 0.7657280564308167,
      "learning_rate": 6.788888888888889e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.175457000732422,
      "logits/rejected": -19.175457000732422,
      "logps/chosen": -0.2413448989391327,
      "logps/rejected": -0.2413448989391327,
      "loss": 5.7839,
      "nll_loss": 1.3766554594039917,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.02413449063897133,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.02413449063897133,
      "step": 390
    },
    {
      "epoch": 0.1446393057313325,
      "grad_norm": 0.7070457935333252,
      "learning_rate": 6.677777777777779e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.15035629272461,
      "logits/rejected": -19.15035629272461,
      "logps/chosen": -0.25096631050109863,
      "logps/rejected": -0.25096631050109863,
      "loss": 6.2307,
      "nll_loss": 1.4883568286895752,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.025096634402871132,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.025096634402871132,
      "step": 400
    },
    {
      "epoch": 0.1482552883746158,
      "grad_norm": 0.43025216460227966,
      "learning_rate": 6.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.02223014831543,
      "logits/rejected": -19.02223014831543,
      "logps/chosen": -0.24732474982738495,
      "logps/rejected": -0.24732474982738495,
      "loss": 6.2865,
      "nll_loss": 1.5023012161254883,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.024732474237680435,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.024732474237680435,
      "step": 410
    },
    {
      "epoch": 0.1518712710178991,
      "grad_norm": 1.3701528310775757,
      "learning_rate": 6.455555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.089460372924805,
      "logits/rejected": -19.089460372924805,
      "logps/chosen": -0.23749932646751404,
      "logps/rejected": -0.23749932646751404,
      "loss": 6.2238,
      "nll_loss": 1.4866377115249634,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.023749932646751404,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.023749932646751404,
      "step": 420
    },
    {
      "epoch": 0.15548725366118243,
      "grad_norm": 0.9421939253807068,
      "learning_rate": 6.3444444444444454e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.231048583984375,
      "logits/rejected": -19.231048583984375,
      "logps/chosen": -0.19140982627868652,
      "logps/rejected": -0.19140982627868652,
      "loss": 6.0814,
      "nll_loss": 1.451047420501709,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01914098486304283,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01914098486304283,
      "step": 430
    },
    {
      "epoch": 0.15910323630446574,
      "grad_norm": 0.9060840010643005,
      "learning_rate": 6.2333333333333335e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.107379913330078,
      "logits/rejected": -19.107379913330078,
      "logps/chosen": -0.2809773087501526,
      "logps/rejected": -0.2809773087501526,
      "loss": 6.3671,
      "nll_loss": 1.522457480430603,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.0280977301299572,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.0280977301299572,
      "step": 440
    },
    {
      "epoch": 0.16271921894774904,
      "grad_norm": 0.5965518355369568,
      "learning_rate": 6.1222222222222224e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.34296417236328,
      "logits/rejected": -19.34296417236328,
      "logps/chosen": -0.19397786259651184,
      "logps/rejected": -0.19397786259651184,
      "loss": 5.7975,
      "nll_loss": 1.3800629377365112,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.019397784024477005,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.019397784024477005,
      "step": 450
    },
    {
      "epoch": 0.16633520159103235,
      "grad_norm": 0.9895502328872681,
      "learning_rate": 6.011111111111112e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.35286521911621,
      "logits/rejected": -19.35286521911621,
      "logps/chosen": -0.20646443963050842,
      "logps/rejected": -0.20646443963050842,
      "loss": 5.8574,
      "nll_loss": 1.3950278759002686,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.02064644545316696,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.02064644545316696,
      "step": 460
    },
    {
      "epoch": 0.16995118423431568,
      "grad_norm": 1.6251282691955566,
      "learning_rate": 5.9e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.29703140258789,
      "logits/rejected": -19.29703140258789,
      "logps/chosen": -0.23044386506080627,
      "logps/rejected": -0.23044386506080627,
      "loss": 6.0743,
      "nll_loss": 1.4492676258087158,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.023044386878609657,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.023044386878609657,
      "step": 470
    },
    {
      "epoch": 0.173567166877599,
      "grad_norm": 0.7959076762199402,
      "learning_rate": 5.788888888888889e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.089004516601562,
      "logits/rejected": -19.089004516601562,
      "logps/chosen": -0.1713530272245407,
      "logps/rejected": -0.1713530272245407,
      "loss": 6.0034,
      "nll_loss": 1.4315412044525146,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01713530346751213,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01713530346751213,
      "step": 480
    },
    {
      "epoch": 0.1771831495208823,
      "grad_norm": 0.6285834908485413,
      "learning_rate": 5.677777777777779e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.485876083374023,
      "logits/rejected": -19.485876083374023,
      "logps/chosen": -0.18361307680606842,
      "logps/rejected": -0.18361307680606842,
      "loss": 5.6858,
      "nll_loss": 1.3521363735198975,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.018361307680606842,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.018361307680606842,
      "step": 490
    },
    {
      "epoch": 0.1807991321641656,
      "grad_norm": 0.9556539058685303,
      "learning_rate": 5.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.300710678100586,
      "logits/rejected": -19.300710678100586,
      "logps/chosen": -0.17280462384223938,
      "logps/rejected": -0.17280462384223938,
      "loss": 6.0335,
      "nll_loss": 1.439051866531372,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01728046126663685,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01728046126663685,
      "step": 500
    },
    {
      "epoch": 0.18441511480744893,
      "grad_norm": 0.7111396789550781,
      "learning_rate": 5.455555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.141691207885742,
      "logits/rejected": -19.141691207885742,
      "logps/chosen": -0.17267338931560516,
      "logps/rejected": -0.17267338931560516,
      "loss": 6.1707,
      "nll_loss": 1.4733521938323975,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.017267340794205666,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.017267340794205666,
      "step": 510
    },
    {
      "epoch": 0.18803109745073224,
      "grad_norm": 1.1565580368041992,
      "learning_rate": 5.344444444444446e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.17026138305664,
      "logits/rejected": -19.17026138305664,
      "logps/chosen": -0.25084125995635986,
      "logps/rejected": -0.25084125995635986,
      "loss": 6.1347,
      "nll_loss": 1.464364767074585,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.025084126740694046,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.025084126740694046,
      "step": 520
    },
    {
      "epoch": 0.19164708009401554,
      "grad_norm": 0.8986483216285706,
      "learning_rate": 5.233333333333334e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.375362396240234,
      "logits/rejected": -19.375362396240234,
      "logps/chosen": -0.24914030730724335,
      "logps/rejected": -0.24914030730724335,
      "loss": 6.1778,
      "nll_loss": 1.475141167640686,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.024914031848311424,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.024914031848311424,
      "step": 530
    },
    {
      "epoch": 0.19526306273729885,
      "grad_norm": 1.2064629793167114,
      "learning_rate": 5.122222222222223e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.600208282470703,
      "logits/rejected": -19.600208282470703,
      "logps/chosen": -0.18557456135749817,
      "logps/rejected": -0.18557456135749817,
      "loss": 5.4563,
      "nll_loss": 1.2947726249694824,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.018557453528046608,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.018557453528046608,
      "step": 540
    },
    {
      "epoch": 0.19887904538058218,
      "grad_norm": 0.6842054128646851,
      "learning_rate": 5.011111111111111e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.55367660522461,
      "logits/rejected": -19.55367660522461,
      "logps/chosen": -0.19609031081199646,
      "logps/rejected": -0.19609031081199646,
      "loss": 6.0472,
      "nll_loss": 1.442474365234375,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.019609034061431885,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.019609034061431885,
      "step": 550
    },
    {
      "epoch": 0.2024950280238655,
      "grad_norm": 0.5375315546989441,
      "learning_rate": 4.9000000000000005e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.148658752441406,
      "logits/rejected": -19.148658752441406,
      "logps/chosen": -0.17001472413539886,
      "logps/rejected": -0.17001472413539886,
      "loss": 5.6006,
      "nll_loss": 1.3308324813842773,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.017001474276185036,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.017001474276185036,
      "step": 560
    },
    {
      "epoch": 0.2061110106671488,
      "grad_norm": 1.169607400894165,
      "learning_rate": 4.7888888888888894e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.166362762451172,
      "logits/rejected": -19.166362762451172,
      "logps/chosen": -0.19456727802753448,
      "logps/rejected": -0.19456727802753448,
      "loss": 6.1838,
      "nll_loss": 1.476626992225647,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01945672743022442,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01945672743022442,
      "step": 570
    },
    {
      "epoch": 0.2097269933104321,
      "grad_norm": 1.1300163269042969,
      "learning_rate": 4.677777777777778e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.370128631591797,
      "logits/rejected": -19.370128631591797,
      "logps/chosen": -0.20290282368659973,
      "logps/rejected": -0.20290282368659973,
      "loss": 5.8201,
      "nll_loss": 1.385710597038269,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.020290281623601913,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.020290281623601913,
      "step": 580
    },
    {
      "epoch": 0.21334297595371543,
      "grad_norm": 1.181201696395874,
      "learning_rate": 4.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.285415649414062,
      "logits/rejected": -19.285415649414062,
      "logps/chosen": -0.18727798759937286,
      "logps/rejected": -0.18727798759937286,
      "loss": 5.8618,
      "nll_loss": 1.3961261510849,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.018727798014879227,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.018727798014879227,
      "step": 590
    },
    {
      "epoch": 0.21695895859699874,
      "grad_norm": 0.8772425651550293,
      "learning_rate": 4.455555555555555e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.32810401916504,
      "logits/rejected": -19.32810401916504,
      "logps/chosen": -0.1583147644996643,
      "logps/rejected": -0.1583147644996643,
      "loss": 5.9621,
      "nll_loss": 1.4212169647216797,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.0158314760774374,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.0158314760774374,
      "step": 600
    },
    {
      "epoch": 0.22057494124028204,
      "grad_norm": 0.9875885248184204,
      "learning_rate": 4.344444444444445e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.371606826782227,
      "logits/rejected": -19.371606826782227,
      "logps/chosen": -0.15842756628990173,
      "logps/rejected": -0.15842756628990173,
      "loss": 5.9785,
      "nll_loss": 1.4253188371658325,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.015842756256461143,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.015842756256461143,
      "step": 610
    },
    {
      "epoch": 0.22419092388356535,
      "grad_norm": 0.8926348090171814,
      "learning_rate": 4.233333333333334e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.365840911865234,
      "logits/rejected": -19.365840911865234,
      "logps/chosen": -0.2225889265537262,
      "logps/rejected": -0.2225889265537262,
      "loss": 5.9332,
      "nll_loss": 1.413989782333374,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.02225889265537262,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.02225889265537262,
      "step": 620
    },
    {
      "epoch": 0.22780690652684868,
      "grad_norm": 1.2021228075027466,
      "learning_rate": 4.122222222222222e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.250947952270508,
      "logits/rejected": -19.250947952270508,
      "logps/chosen": -0.20725078880786896,
      "logps/rejected": -0.20725078880786896,
      "loss": 5.8929,
      "nll_loss": 1.4039117097854614,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.020725077018141747,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.020725077018141747,
      "step": 630
    },
    {
      "epoch": 0.23142288917013198,
      "grad_norm": 0.6190844774246216,
      "learning_rate": 4.011111111111111e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.514095306396484,
      "logits/rejected": -19.514095306396484,
      "logps/chosen": -0.14317741990089417,
      "logps/rejected": -0.14317741990089417,
      "loss": 5.655,
      "nll_loss": 1.3444368839263916,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.014317741617560387,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.014317741617560387,
      "step": 640
    },
    {
      "epoch": 0.2350388718134153,
      "grad_norm": 0.6763967871665955,
      "learning_rate": 3.900000000000001e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.486291885375977,
      "logits/rejected": -19.486291885375977,
      "logps/chosen": -0.19646582007408142,
      "logps/rejected": -0.19646582007408142,
      "loss": 6.1549,
      "nll_loss": 1.4694058895111084,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.019646581262350082,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.019646581262350082,
      "step": 650
    },
    {
      "epoch": 0.2386548544566986,
      "grad_norm": 1.276583194732666,
      "learning_rate": 3.7888888888888893e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.337284088134766,
      "logits/rejected": -19.337284088134766,
      "logps/chosen": -0.16177110373973846,
      "logps/rejected": -0.16177110373973846,
      "loss": 6.0415,
      "nll_loss": 1.441070318222046,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.016177110373973846,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.016177110373973846,
      "step": 660
    },
    {
      "epoch": 0.24227083709998193,
      "grad_norm": 1.4721999168395996,
      "learning_rate": 3.6777777777777778e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.851795196533203,
      "logits/rejected": -19.851795196533203,
      "logps/chosen": -0.19354157149791718,
      "logps/rejected": -0.19354157149791718,
      "loss": 6.0983,
      "nll_loss": 1.455251932144165,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.019354157149791718,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.019354157149791718,
      "step": 670
    },
    {
      "epoch": 0.24588681974326523,
      "grad_norm": 1.205200433731079,
      "learning_rate": 3.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.199329376220703,
      "logits/rejected": -19.199329376220703,
      "logps/chosen": -0.2025156319141388,
      "logps/rejected": -0.2025156319141388,
      "loss": 6.0431,
      "nll_loss": 1.441453218460083,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.0202515609562397,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.0202515609562397,
      "step": 680
    },
    {
      "epoch": 0.24950280238654854,
      "grad_norm": 0.4968813359737396,
      "learning_rate": 3.455555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.286746978759766,
      "logits/rejected": -19.286746978759766,
      "logps/chosen": -0.20702144503593445,
      "logps/rejected": -0.20702144503593445,
      "loss": 6.0379,
      "nll_loss": 1.4401479959487915,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.020702145993709564,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.020702145993709564,
      "step": 690
    },
    {
      "epoch": 0.25311878502983187,
      "grad_norm": 0.49326029419898987,
      "learning_rate": 3.3444444444444445e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.63970184326172,
      "logits/rejected": -19.63970184326172,
      "logps/chosen": -0.1862333118915558,
      "logps/rejected": -0.1862333118915558,
      "loss": 5.6812,
      "nll_loss": 1.350982666015625,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.018623333424329758,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.018623333424329758,
      "step": 700
    },
    {
      "epoch": 0.25673476767311515,
      "grad_norm": 1.1381062269210815,
      "learning_rate": 3.2333333333333334e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.339035034179688,
      "logits/rejected": -19.339035034179688,
      "logps/chosen": -0.22874228656291962,
      "logps/rejected": -0.22874228656291962,
      "loss": 6.5147,
      "nll_loss": 1.5593668222427368,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.022874226793646812,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.022874226793646812,
      "step": 710
    },
    {
      "epoch": 0.2603507503163985,
      "grad_norm": 1.3999911546707153,
      "learning_rate": 3.1222222222222228e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.305469512939453,
      "logits/rejected": -19.305469512939453,
      "logps/chosen": -0.1623169630765915,
      "logps/rejected": -0.1623169630765915,
      "loss": 5.9032,
      "nll_loss": 1.4064949750900269,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01623169705271721,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01623169705271721,
      "step": 720
    },
    {
      "epoch": 0.2639667329596818,
      "grad_norm": 0.6383623480796814,
      "learning_rate": 3.0111111111111113e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.534765243530273,
      "logits/rejected": -19.534765243530273,
      "logps/chosen": -0.15669655799865723,
      "logps/rejected": -0.15669655799865723,
      "loss": 5.723,
      "nll_loss": 1.3614407777786255,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.015669656917452812,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.015669656917452812,
      "step": 730
    },
    {
      "epoch": 0.2675827156029651,
      "grad_norm": 1.1800575256347656,
      "learning_rate": 2.9e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.569406509399414,
      "logits/rejected": -19.569406509399414,
      "logps/chosen": -0.17918285727500916,
      "logps/rejected": -0.17918285727500916,
      "loss": 5.7004,
      "nll_loss": 1.355780839920044,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.017918284982442856,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.017918284982442856,
      "step": 740
    },
    {
      "epoch": 0.2711986982462484,
      "grad_norm": 0.5818102359771729,
      "learning_rate": 2.788888888888889e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.516826629638672,
      "logits/rejected": -19.516826629638672,
      "logps/chosen": -0.14605708420276642,
      "logps/rejected": -0.14605708420276642,
      "loss": 5.8445,
      "nll_loss": 1.391806960105896,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.014605708420276642,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.014605708420276642,
      "step": 750
    },
    {
      "epoch": 0.2748146808895317,
      "grad_norm": 0.6804122924804688,
      "learning_rate": 2.677777777777778e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.40943145751953,
      "logits/rejected": -19.40943145751953,
      "logps/chosen": -0.17038683593273163,
      "logps/rejected": -0.17038683593273163,
      "loss": 5.8438,
      "nll_loss": 1.391636848449707,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.017038684338331223,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.017038684338331223,
      "step": 760
    },
    {
      "epoch": 0.27843066353281504,
      "grad_norm": 0.7628147006034851,
      "learning_rate": 2.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.423494338989258,
      "logits/rejected": -19.423494338989258,
      "logps/chosen": -0.20526845753192902,
      "logps/rejected": -0.20526845753192902,
      "loss": 6.2205,
      "nll_loss": 1.4858171939849854,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.02052684687077999,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.02052684687077999,
      "step": 770
    },
    {
      "epoch": 0.28204664617609837,
      "grad_norm": 0.6365695595741272,
      "learning_rate": 2.455555555555556e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.535381317138672,
      "logits/rejected": -19.535381317138672,
      "logps/chosen": -0.138637512922287,
      "logps/rejected": -0.138637512922287,
      "loss": 5.848,
      "nll_loss": 1.392683744430542,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.013863752596080303,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.013863752596080303,
      "step": 780
    },
    {
      "epoch": 0.28566262881938165,
      "grad_norm": 0.4407297968864441,
      "learning_rate": 2.3444444444444448e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.312089920043945,
      "logits/rejected": -19.312089920043945,
      "logps/chosen": -0.21358434855937958,
      "logps/rejected": -0.21358434855937958,
      "loss": 5.8644,
      "nll_loss": 1.396776795387268,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.021358435973525047,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.021358435973525047,
      "step": 790
    },
    {
      "epoch": 0.289278611462665,
      "grad_norm": 1.0635472536087036,
      "learning_rate": 2.2333333333333333e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.445270538330078,
      "logits/rejected": -19.445270538330078,
      "logps/chosen": -0.14682866632938385,
      "logps/rejected": -0.14682866632938385,
      "loss": 5.5576,
      "nll_loss": 1.3200775384902954,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.014682864770293236,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.014682864770293236,
      "step": 800
    },
    {
      "epoch": 0.2928945941059483,
      "grad_norm": 0.6877497434616089,
      "learning_rate": 2.1222222222222226e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.4804744720459,
      "logits/rejected": -19.4804744720459,
      "logps/chosen": -0.20306849479675293,
      "logps/rejected": -0.20306849479675293,
      "loss": 6.2889,
      "nll_loss": 1.5029165744781494,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.020306849852204323,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.020306849852204323,
      "step": 810
    },
    {
      "epoch": 0.2965105767492316,
      "grad_norm": 0.7547898888587952,
      "learning_rate": 2.011111111111111e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.336137771606445,
      "logits/rejected": -19.336137771606445,
      "logps/chosen": -0.18079710006713867,
      "logps/rejected": -0.18079710006713867,
      "loss": 5.777,
      "nll_loss": 1.3749300241470337,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.018079709261655807,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.018079709261655807,
      "step": 820
    },
    {
      "epoch": 0.3001265593925149,
      "grad_norm": 1.389726996421814,
      "learning_rate": 1.9000000000000002e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.307830810546875,
      "logits/rejected": -19.307830810546875,
      "logps/chosen": -0.15548691153526306,
      "logps/rejected": -0.15548691153526306,
      "loss": 6.3126,
      "nll_loss": 1.5088294744491577,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.015548691153526306,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.015548691153526306,
      "step": 830
    },
    {
      "epoch": 0.3037425420357982,
      "grad_norm": 0.6280723810195923,
      "learning_rate": 1.788888888888889e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.246450424194336,
      "logits/rejected": -19.246450424194336,
      "logps/chosen": -0.19246172904968262,
      "logps/rejected": -0.19246172904968262,
      "loss": 6.0085,
      "nll_loss": 1.4328036308288574,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01924617402255535,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01924617402255535,
      "step": 840
    },
    {
      "epoch": 0.30735852467908154,
      "grad_norm": 0.9361159205436707,
      "learning_rate": 1.6777777777777779e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.374317169189453,
      "logits/rejected": -19.374317169189453,
      "logps/chosen": -0.20890088379383087,
      "logps/rejected": -0.20890088379383087,
      "loss": 5.9353,
      "nll_loss": 1.4145065546035767,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.020890090614557266,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.020890090614557266,
      "step": 850
    },
    {
      "epoch": 0.31097450732236487,
      "grad_norm": 0.8236832022666931,
      "learning_rate": 1.566666666666667e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.475786209106445,
      "logits/rejected": -19.475786209106445,
      "logps/chosen": -0.1413622796535492,
      "logps/rejected": -0.1413622796535492,
      "loss": 5.9494,
      "nll_loss": 1.4180256128311157,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.014136227779090405,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.014136227779090405,
      "step": 860
    },
    {
      "epoch": 0.31459048996564815,
      "grad_norm": 0.7201619148254395,
      "learning_rate": 1.4555555555555557e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.682842254638672,
      "logits/rejected": -19.682842254638672,
      "logps/chosen": -0.18202020227909088,
      "logps/rejected": -0.18202020227909088,
      "loss": 5.9195,
      "nll_loss": 1.410556674003601,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01820201985538006,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01820201985538006,
      "step": 870
    },
    {
      "epoch": 0.3182064726089315,
      "grad_norm": 0.886303722858429,
      "learning_rate": 1.3444444444444446e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.420122146606445,
      "logits/rejected": -19.420122146606445,
      "logps/chosen": -0.17583726346492767,
      "logps/rejected": -0.17583726346492767,
      "loss": 5.5973,
      "nll_loss": 1.3300113677978516,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.017583727836608887,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.017583727836608887,
      "step": 880
    },
    {
      "epoch": 0.3218224552522148,
      "grad_norm": 0.8513539433479309,
      "learning_rate": 1.2444444444444445e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.468677520751953,
      "logits/rejected": -19.468677520751953,
      "logps/chosen": -0.16836388409137726,
      "logps/rejected": -0.16836388409137726,
      "loss": 6.062,
      "nll_loss": 1.4461798667907715,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.016836389899253845,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.016836389899253845,
      "step": 890
    },
    {
      "epoch": 0.3254384378954981,
      "grad_norm": 1.0691941976547241,
      "learning_rate": 1.1333333333333334e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.57880210876465,
      "logits/rejected": -19.57880210876465,
      "logps/chosen": -0.16049222648143768,
      "logps/rejected": -0.16049222648143768,
      "loss": 5.9859,
      "nll_loss": 1.4271578788757324,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01604922115802765,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01604922115802765,
      "step": 900
    },
    {
      "epoch": 0.3290544205387814,
      "grad_norm": 1.168095350265503,
      "learning_rate": 1.0222222222222223e-06,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.2463436126709,
      "logits/rejected": -19.2463436126709,
      "logps/chosen": -0.1521718055009842,
      "logps/rejected": -0.1521718055009842,
      "loss": 5.71,
      "nll_loss": 1.3581907749176025,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.015217180363833904,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.015217180363833904,
      "step": 910
    },
    {
      "epoch": 0.3326704031820647,
      "grad_norm": 0.862614631652832,
      "learning_rate": 9.111111111111113e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.247844696044922,
      "logits/rejected": -19.247844696044922,
      "logps/chosen": -0.19881650805473328,
      "logps/rejected": -0.19881650805473328,
      "loss": 5.9249,
      "nll_loss": 1.4119006395339966,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.019881650805473328,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.019881650805473328,
      "step": 920
    },
    {
      "epoch": 0.33628638582534803,
      "grad_norm": 0.9760550260543823,
      "learning_rate": 8.000000000000001e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.753040313720703,
      "logits/rejected": -19.753040313720703,
      "logps/chosen": -0.17076358199119568,
      "logps/rejected": -0.17076358199119568,
      "loss": 5.8841,
      "nll_loss": 1.401706337928772,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.017076358199119568,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.017076358199119568,
      "step": 930
    },
    {
      "epoch": 0.33990236846863137,
      "grad_norm": 0.8302357196807861,
      "learning_rate": 6.88888888888889e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.467557907104492,
      "logits/rejected": -19.467557907104492,
      "logps/chosen": -0.15472976863384247,
      "logps/rejected": -0.15472976863384247,
      "loss": 5.5121,
      "nll_loss": 1.308708906173706,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.015472980216145515,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.015472980216145515,
      "step": 940
    },
    {
      "epoch": 0.34351835111191464,
      "grad_norm": 1.4063400030136108,
      "learning_rate": 5.777777777777778e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.383520126342773,
      "logits/rejected": -19.383520126342773,
      "logps/chosen": -0.16120824217796326,
      "logps/rejected": -0.16120824217796326,
      "loss": 6.0657,
      "nll_loss": 1.4471219778060913,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.016120824962854385,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.016120824962854385,
      "step": 950
    },
    {
      "epoch": 0.347134333755198,
      "grad_norm": 1.351473331451416,
      "learning_rate": 4.666666666666667e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.597942352294922,
      "logits/rejected": -19.597942352294922,
      "logps/chosen": -0.1628998965024948,
      "logps/rejected": -0.1628998965024948,
      "loss": 5.8956,
      "nll_loss": 1.4045830965042114,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01628998853266239,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01628998853266239,
      "step": 960
    },
    {
      "epoch": 0.3507503163984813,
      "grad_norm": 0.688176691532135,
      "learning_rate": 3.555555555555556e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.574413299560547,
      "logits/rejected": -19.574413299560547,
      "logps/chosen": -0.12833839654922485,
      "logps/rejected": -0.12833839654922485,
      "loss": 5.6608,
      "nll_loss": 1.3458738327026367,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.012833841145038605,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.012833841145038605,
      "step": 970
    },
    {
      "epoch": 0.3543662990417646,
      "grad_norm": 0.605377733707428,
      "learning_rate": 2.444444444444445e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -20.091583251953125,
      "logits/rejected": -20.091583251953125,
      "logps/chosen": -0.1359986811876297,
      "logps/rejected": -0.1359986811876297,
      "loss": 5.6425,
      "nll_loss": 1.3413007259368896,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.013599867932498455,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.013599867932498455,
      "step": 980
    },
    {
      "epoch": 0.3579822816850479,
      "grad_norm": 1.1121692657470703,
      "learning_rate": 1.3333333333333336e-07,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.476083755493164,
      "logits/rejected": -19.476083755493164,
      "logps/chosen": -0.1606629192829132,
      "logps/rejected": -0.1606629192829132,
      "loss": 5.9177,
      "nll_loss": 1.4101154804229736,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.01606629230082035,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.01606629230082035,
      "step": 990
    },
    {
      "epoch": 0.3615982643283312,
      "grad_norm": 0.8480359315872192,
      "learning_rate": 2.2222222222222224e-08,
      "log_odds_chosen": 0.0,
      "log_odds_ratio": -0.6931472420692444,
      "logits/chosen": -19.61067771911621,
      "logits/rejected": -19.61067771911621,
      "logps/chosen": -0.14116862416267395,
      "logps/rejected": -0.14116862416267395,
      "loss": 5.7495,
      "nll_loss": 1.3680593967437744,
      "rewards/accuracies": 0.0,
      "rewards/chosen": -0.014116862788796425,
      "rewards/margins": 0.0,
      "rewards/rejected": -0.014116862788796425,
      "step": 1000
    }
  ],
  "logging_steps": 10,
  "max_steps": 1000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}