{
  "best_metric": 0.3940436840057373,
  "best_model_checkpoint": "./mistral/19-04-24-Weni-WeniGPT-Agents-Mistral-1.0.6-SFT-1.0.3-DPO_Experiment on DPO with other hyperparameters and best SFT model of WeniGPT-2_max_steps-366_batch_4_2024-04-19_ppid_9/checkpoint-360",
  "epoch": 5.853658536585366,
  "eval_steps": 30,
  "global_step": 360,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.16,
      "grad_norm": 29.063692092895508,
      "learning_rate": 3.181818181818182e-06,
      "logits/chosen": -1.7455532550811768,
      "logits/rejected": -1.7793405055999756,
      "logps/chosen": -54.72136306762695,
      "logps/rejected": -97.28812408447266,
      "loss": 0.6924,
      "rewards/accuracies": 0.15000000596046448,
      "rewards/chosen": 0.003122234484180808,
      "rewards/margins": 0.0015356539515778422,
      "rewards/rejected": 0.0015865802997723222,
      "step": 10
    },
    {
      "epoch": 0.33,
      "grad_norm": 15.157337188720703,
      "learning_rate": 4.9154929577464795e-06,
      "logits/chosen": -1.8641210794448853,
      "logits/rejected": -1.8705828189849854,
      "logps/chosen": -40.3618049621582,
      "logps/rejected": -55.96257400512695,
      "loss": 0.6737,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.04275550693273544,
      "rewards/margins": 0.040880247950553894,
      "rewards/rejected": 0.0018752537434920669,
      "step": 20
    },
    {
      "epoch": 0.49,
      "grad_norm": 13.148392677307129,
      "learning_rate": 4.774647887323944e-06,
      "logits/chosen": -1.8284633159637451,
      "logits/rejected": -1.8519256114959717,
      "logps/chosen": -44.69489288330078,
      "logps/rejected": -76.79936981201172,
      "loss": 0.6371,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.12410124391317368,
      "rewards/margins": 0.1248731017112732,
      "rewards/rejected": -0.0007718756678514183,
      "step": 30
    },
    {
      "epoch": 0.49,
      "eval_logits/chosen": -1.7379764318466187,
      "eval_logits/rejected": -1.7726188898086548,
      "eval_logps/chosen": -50.49373245239258,
      "eval_logps/rejected": -83.54340362548828,
      "eval_loss": 0.5864880681037903,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 0.24716782569885254,
      "eval_rewards/margins": 0.25581416487693787,
      "eval_rewards/rejected": -0.008646338246762753,
      "eval_runtime": 8.1282,
      "eval_samples_per_second": 3.445,
      "eval_steps_per_second": 1.722,
      "step": 30
    },
    {
      "epoch": 0.65,
      "grad_norm": 24.143388748168945,
      "learning_rate": 4.633802816901409e-06,
      "logits/chosen": -1.8080145120620728,
      "logits/rejected": -1.8319604396820068,
      "logps/chosen": -46.848201751708984,
      "logps/rejected": -81.42877960205078,
      "loss": 0.5649,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.3360167443752289,
      "rewards/margins": 0.34645381569862366,
      "rewards/rejected": -0.010437069460749626,
      "step": 40
    },
    {
      "epoch": 0.81,
      "grad_norm": 0.0,
      "learning_rate": 4.492957746478874e-06,
      "logits/chosen": -1.8638660907745361,
      "logits/rejected": -1.8754875659942627,
      "logps/chosen": -22.06886863708496,
      "logps/rejected": -41.17422103881836,
      "loss": 0.6377,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": 0.25195080041885376,
      "rewards/margins": 0.16642150282859802,
      "rewards/rejected": 0.08552931249141693,
      "step": 50
    },
    {
      "epoch": 0.98,
      "grad_norm": 7.51575231552124,
      "learning_rate": 4.352112676056338e-06,
      "logits/chosen": -1.8215672969818115,
      "logits/rejected": -1.8419084548950195,
      "logps/chosen": -44.14409255981445,
      "logps/rejected": -77.38871002197266,
      "loss": 0.5496,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.4203863739967346,
      "rewards/margins": 0.458610475063324,
      "rewards/rejected": -0.038224123418331146,
      "step": 60
    },
    {
      "epoch": 0.98,
      "eval_logits/chosen": -1.7419661283493042,
      "eval_logits/rejected": -1.7773857116699219,
      "eval_logps/chosen": -49.36268997192383,
      "eval_logps/rejected": -83.60606384277344,
      "eval_loss": 0.4963766038417816,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 0.5864797234535217,
      "eval_rewards/margins": 0.613922655582428,
      "eval_rewards/rejected": -0.02744293212890625,
      "eval_runtime": 8.1315,
      "eval_samples_per_second": 3.443,
      "eval_steps_per_second": 1.722,
      "step": 60
    },
    {
      "epoch": 1.14,
      "grad_norm": 3.454394817352295,
      "learning_rate": 4.211267605633803e-06,
      "logits/chosen": -1.7844343185424805,
      "logits/rejected": -1.8126220703125,
      "logps/chosen": -39.737998962402344,
      "logps/rejected": -78.76033782958984,
      "loss": 0.4925,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.6417733430862427,
      "rewards/margins": 0.7573493123054504,
      "rewards/rejected": -0.11557593196630478,
      "step": 70
    },
    {
      "epoch": 1.3,
      "grad_norm": 14.345088005065918,
      "learning_rate": 4.070422535211268e-06,
      "logits/chosen": -1.822148084640503,
      "logits/rejected": -1.8463541269302368,
      "logps/chosen": -35.735836029052734,
      "logps/rejected": -75.4683837890625,
      "loss": 0.4965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.6439998745918274,
      "rewards/margins": 0.8037986755371094,
      "rewards/rejected": -0.1597987860441208,
      "step": 80
    },
    {
      "epoch": 1.46,
      "grad_norm": 12.988151550292969,
      "learning_rate": 3.9295774647887325e-06,
      "logits/chosen": -1.858496069908142,
      "logits/rejected": -1.8683990240097046,
      "logps/chosen": -30.166736602783203,
      "logps/rejected": -50.970542907714844,
      "loss": 0.5185,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.6545813679695129,
      "rewards/margins": 0.7556509971618652,
      "rewards/rejected": -0.10106959193944931,
      "step": 90
    },
    {
      "epoch": 1.46,
      "eval_logits/chosen": -1.7461166381835938,
      "eval_logits/rejected": -1.7827314138412476,
      "eval_logps/chosen": -47.9538688659668,
      "eval_logps/rejected": -83.84149169921875,
      "eval_loss": 0.4401554763317108,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 1.0091263055801392,
      "eval_rewards/margins": 1.1071958541870117,
      "eval_rewards/rejected": -0.09806957095861435,
      "eval_runtime": 8.1286,
      "eval_samples_per_second": 3.445,
      "eval_steps_per_second": 1.722,
      "step": 90
    },
    {
      "epoch": 1.63,
      "grad_norm": 7.574762344360352,
      "learning_rate": 3.7887323943661976e-06,
      "logits/chosen": -1.8498780727386475,
      "logits/rejected": -1.8676494359970093,
      "logps/chosen": -50.50703811645508,
      "logps/rejected": -83.80486297607422,
      "loss": 0.4752,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 1.088062047958374,
      "rewards/margins": 1.0784289836883545,
      "rewards/rejected": 0.009633284993469715,
      "step": 100
    },
    {
      "epoch": 1.79,
      "grad_norm": 10.141860008239746,
      "learning_rate": 3.6478873239436626e-06,
      "logits/chosen": -1.8536640405654907,
      "logits/rejected": -1.8669731616973877,
      "logps/chosen": -33.00554275512695,
      "logps/rejected": -46.077144622802734,
      "loss": 0.5439,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.7643357515335083,
      "rewards/margins": 0.7525759339332581,
      "rewards/rejected": 0.011759823188185692,
      "step": 110
    },
    {
      "epoch": 1.95,
      "grad_norm": 0.0,
      "learning_rate": 3.5070422535211268e-06,
      "logits/chosen": -1.8447682857513428,
      "logits/rejected": -1.8607944250106812,
      "logps/chosen": -42.43408966064453,
      "logps/rejected": -73.089599609375,
      "loss": 0.4623,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 1.1391878128051758,
      "rewards/margins": 1.3124778270721436,
      "rewards/rejected": -0.17328998446464539,
      "step": 120
    },
    {
      "epoch": 1.95,
      "eval_logits/chosen": -1.7511886358261108,
      "eval_logits/rejected": -1.7883756160736084,
      "eval_logps/chosen": -46.9849739074707,
      "eval_logps/rejected": -84.11779022216797,
      "eval_loss": 0.4216788709163666,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 1.2997931241989136,
      "eval_rewards/margins": 1.4807531833648682,
      "eval_rewards/rejected": -0.1809600442647934,
      "eval_runtime": 8.133,
      "eval_samples_per_second": 3.443,
      "eval_steps_per_second": 1.721,
      "step": 120
    },
    {
      "epoch": 2.11,
      "grad_norm": 0.7550886869430542,
      "learning_rate": 3.3661971830985918e-06,
      "logits/chosen": -1.765539526939392,
      "logits/rejected": -1.8175758123397827,
      "logps/chosen": -50.52437973022461,
      "logps/rejected": -116.59224700927734,
      "loss": 0.3356,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 1.8601114749908447,
      "rewards/margins": 2.246537685394287,
      "rewards/rejected": -0.3864262104034424,
      "step": 130
    },
    {
      "epoch": 2.28,
      "grad_norm": 0.23429778218269348,
      "learning_rate": 3.225352112676057e-06,
      "logits/chosen": -1.7902629375457764,
      "logits/rejected": -1.8166630268096924,
      "logps/chosen": -41.8301887512207,
      "logps/rejected": -78.8285140991211,
      "loss": 0.4254,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 1.3376247882843018,
      "rewards/margins": 1.733752965927124,
      "rewards/rejected": -0.39612799882888794,
      "step": 140
    },
    {
      "epoch": 2.44,
      "grad_norm": 0.4628850817680359,
      "learning_rate": 3.084507042253521e-06,
      "logits/chosen": -1.8713032007217407,
      "logits/rejected": -1.8906549215316772,
      "logps/chosen": -38.51209259033203,
      "logps/rejected": -69.11054992675781,
      "loss": 0.4985,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 1.159945011138916,
      "rewards/margins": 1.5182414054870605,
      "rewards/rejected": -0.35829633474349976,
      "step": 150
    },
    {
      "epoch": 2.44,
      "eval_logits/chosen": -1.7590656280517578,
      "eval_logits/rejected": -1.7967649698257446,
      "eval_logps/chosen": -45.99831008911133,
      "eval_logps/rejected": -84.5901107788086,
      "eval_loss": 0.40688762068748474,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 1.5957926511764526,
      "eval_rewards/margins": 1.9184513092041016,
      "eval_rewards/rejected": -0.3226587474346161,
      "eval_runtime": 8.13,
      "eval_samples_per_second": 3.444,
      "eval_steps_per_second": 1.722,
      "step": 150
    },
    {
      "epoch": 2.6,
      "grad_norm": 2.2907166481018066,
      "learning_rate": 2.943661971830986e-06,
      "logits/chosen": -1.9065347909927368,
      "logits/rejected": -1.9184277057647705,
      "logps/chosen": -35.44993209838867,
      "logps/rejected": -58.14739227294922,
      "loss": 0.5111,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 1.2248560190200806,
      "rewards/margins": 1.3947786092758179,
      "rewards/rejected": -0.1699226349592209,
      "step": 160
    },
    {
      "epoch": 2.76,
      "grad_norm": 0.19129106402397156,
      "learning_rate": 2.802816901408451e-06,
      "logits/chosen": -1.7800159454345703,
      "logits/rejected": -1.8162111043930054,
      "logps/chosen": -44.746212005615234,
      "logps/rejected": -87.70552062988281,
      "loss": 0.3875,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 1.9709749221801758,
      "rewards/margins": 2.394925355911255,
      "rewards/rejected": -0.42395058274269104,
      "step": 170
    },
    {
      "epoch": 2.93,
      "grad_norm": 0.09690173715353012,
      "learning_rate": 2.6619718309859156e-06,
      "logits/chosen": -1.9381850957870483,
      "logits/rejected": -1.9563757181167603,
      "logps/chosen": -20.314172744750977,
      "logps/rejected": -45.916996002197266,
      "loss": 0.5276,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.9397296905517578,
      "rewards/margins": 1.2702549695968628,
      "rewards/rejected": -0.33052533864974976,
      "step": 180
    },
    {
      "epoch": 2.93,
      "eval_logits/chosen": -1.7637720108032227,
      "eval_logits/rejected": -1.8018139600753784,
      "eval_logps/chosen": -45.44319534301758,
      "eval_logps/rejected": -84.9321517944336,
      "eval_loss": 0.40118029713630676,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 1.762328028678894,
      "eval_rewards/margins": 2.1875970363616943,
      "eval_rewards/rejected": -0.425269216299057,
      "eval_runtime": 8.1311,
      "eval_samples_per_second": 3.444,
      "eval_steps_per_second": 1.722,
      "step": 180
    },
    {
      "epoch": 3.09,
      "grad_norm": 9.36091136932373,
      "learning_rate": 2.5211267605633806e-06,
      "logits/chosen": -1.8772424459457397,
      "logits/rejected": -1.8817323446273804,
      "logps/chosen": -24.47315788269043,
      "logps/rejected": -39.80836486816406,
      "loss": 0.5178,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.9229831695556641,
      "rewards/margins": 1.3109080791473389,
      "rewards/rejected": -0.38792505860328674,
      "step": 190
    },
    {
      "epoch": 3.25,
      "grad_norm": 1.1417157649993896,
      "learning_rate": 2.3802816901408452e-06,
      "logits/chosen": -1.8643680810928345,
      "logits/rejected": -1.8863565921783447,
      "logps/chosen": -22.632598876953125,
      "logps/rejected": -55.38536834716797,
      "loss": 0.5328,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 1.202460527420044,
      "rewards/margins": 1.351302146911621,
      "rewards/rejected": -0.1488417685031891,
      "step": 200
    },
    {
      "epoch": 3.41,
      "grad_norm": 3.8279953002929688,
      "learning_rate": 2.2394366197183103e-06,
      "logits/chosen": -1.8616552352905273,
      "logits/rejected": -1.8696205615997314,
      "logps/chosen": -26.352832794189453,
      "logps/rejected": -42.96009063720703,
      "loss": 0.5059,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": 1.3207988739013672,
      "rewards/margins": 1.787398099899292,
      "rewards/rejected": -0.4665992856025696,
      "step": 210
    },
    {
      "epoch": 3.41,
      "eval_logits/chosen": -1.7637497186660767,
      "eval_logits/rejected": -1.8021754026412964,
      "eval_logps/chosen": -45.08576202392578,
      "eval_logps/rejected": -85.06809997558594,
      "eval_loss": 0.39926525950431824,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 1.8695563077926636,
      "eval_rewards/margins": 2.33561110496521,
      "eval_rewards/rejected": -0.46605488657951355,
      "eval_runtime": 8.1323,
      "eval_samples_per_second": 3.443,
      "eval_steps_per_second": 1.722,
      "step": 210
    },
    {
      "epoch": 3.58,
      "grad_norm": 0.5771293044090271,
      "learning_rate": 2.098591549295775e-06,
      "logits/chosen": -1.7401905059814453,
      "logits/rejected": -1.7726894617080688,
      "logps/chosen": -59.92805099487305,
      "logps/rejected": -111.60662841796875,
      "loss": 0.3064,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 2.7733664512634277,
      "rewards/margins": 3.541491746902466,
      "rewards/rejected": -0.7681252360343933,
      "step": 220
    },
    {
      "epoch": 3.74,
      "grad_norm": 0.03903874009847641,
      "learning_rate": 1.9577464788732395e-06,
      "logits/chosen": -1.8807846307754517,
      "logits/rejected": -1.900983214378357,
      "logps/chosen": -36.41189956665039,
      "logps/rejected": -73.43792724609375,
      "loss": 0.4857,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 1.912959098815918,
      "rewards/margins": 2.3472061157226562,
      "rewards/rejected": -0.4342469274997711,
      "step": 230
    },
    {
      "epoch": 3.9,
      "grad_norm": 7.481461524963379,
      "learning_rate": 1.8169014084507045e-06,
      "logits/chosen": -1.8599838018417358,
      "logits/rejected": -1.8959333896636963,
      "logps/chosen": -37.08423614501953,
      "logps/rejected": -75.9119644165039,
      "loss": 0.4308,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 1.5218284130096436,
      "rewards/margins": 2.266434669494629,
      "rewards/rejected": -0.7446064352989197,
      "step": 240
    },
    {
      "epoch": 3.9,
      "eval_logits/chosen": -1.7680739164352417,
      "eval_logits/rejected": -1.8067959547042847,
      "eval_logps/chosen": -44.73001480102539,
      "eval_logps/rejected": -85.37883758544922,
      "eval_loss": 0.3971562683582306,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 1.9762812852859497,
      "eval_rewards/margins": 2.5355570316314697,
      "eval_rewards/rejected": -0.5592758059501648,
      "eval_runtime": 8.1317,
      "eval_samples_per_second": 3.443,
      "eval_steps_per_second": 1.722,
      "step": 240
    },
    {
      "epoch": 4.07,
      "grad_norm": 14.042320251464844,
      "learning_rate": 1.676056338028169e-06,
      "logits/chosen": -1.8893840312957764,
      "logits/rejected": -1.906826376914978,
      "logps/chosen": -37.212867736816406,
      "logps/rejected": -72.95481872558594,
      "loss": 0.4421,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 1.7997238636016846,
      "rewards/margins": 2.3047571182250977,
      "rewards/rejected": -0.5050331354141235,
      "step": 250
    },
    {
      "epoch": 4.23,
      "grad_norm": 0.7587959170341492,
      "learning_rate": 1.5352112676056339e-06,
      "logits/chosen": -1.8049319982528687,
      "logits/rejected": -1.848043441772461,
      "logps/chosen": -41.58928298950195,
      "logps/rejected": -108.19120025634766,
      "loss": 0.4398,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 1.4899482727050781,
      "rewards/margins": 2.264251470565796,
      "rewards/rejected": -0.7743033766746521,
      "step": 260
    },
    {
      "epoch": 4.39,
      "grad_norm": 0.06431450694799423,
      "learning_rate": 1.3943661971830987e-06,
      "logits/chosen": -1.892024278640747,
      "logits/rejected": -1.9154161214828491,
      "logps/chosen": -34.56536102294922,
      "logps/rejected": -67.05437469482422,
      "loss": 0.4277,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 2.181830883026123,
      "rewards/margins": 2.7015929222106934,
      "rewards/rejected": -0.5197622776031494,
      "step": 270
    },
    {
      "epoch": 4.39,
      "eval_logits/chosen": -1.771107792854309,
      "eval_logits/rejected": -1.8099576234817505,
      "eval_logps/chosen": -44.553104400634766,
      "eval_logps/rejected": -85.62330627441406,
      "eval_loss": 0.39542675018310547,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 2.029355764389038,
      "eval_rewards/margins": 2.661973237991333,
      "eval_rewards/rejected": -0.632617175579071,
      "eval_runtime": 8.1303,
      "eval_samples_per_second": 3.444,
      "eval_steps_per_second": 1.722,
      "step": 270
    },
    {
      "epoch": 4.55,
      "grad_norm": 0.8766132593154907,
      "learning_rate": 1.2535211267605635e-06,
      "logits/chosen": -1.9151760339736938,
      "logits/rejected": -1.9344974756240845,
      "logps/chosen": -23.768444061279297,
      "logps/rejected": -53.511009216308594,
      "loss": 0.4885,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 1.5792860984802246,
      "rewards/margins": 2.160665512084961,
      "rewards/rejected": -0.5813794136047363,
      "step": 280
    },
    {
      "epoch": 4.72,
      "grad_norm": 0.4080682694911957,
      "learning_rate": 1.1126760563380283e-06,
      "logits/chosen": -1.846788763999939,
      "logits/rejected": -1.8602848052978516,
      "logps/chosen": -42.09468460083008,
      "logps/rejected": -71.29743194580078,
      "loss": 0.4409,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 1.980571985244751,
      "rewards/margins": 2.5399861335754395,
      "rewards/rejected": -0.5594138503074646,
      "step": 290
    },
    {
      "epoch": 4.88,
      "grad_norm": 0.5663866400718689,
      "learning_rate": 9.71830985915493e-07,
      "logits/chosen": -1.7880895137786865,
      "logits/rejected": -1.8192650079727173,
      "logps/chosen": -38.058753967285156,
      "logps/rejected": -75.80699157714844,
      "loss": 0.4366,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 2.0137667655944824,
      "rewards/margins": 3.0218002796173096,
      "rewards/rejected": -1.0080333948135376,
      "step": 300
    },
    {
      "epoch": 4.88,
      "eval_logits/chosen": -1.7717950344085693,
      "eval_logits/rejected": -1.8107479810714722,
      "eval_logps/chosen": -44.3961067199707,
      "eval_logps/rejected": -85.71527862548828,
      "eval_loss": 0.39505863189697266,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 2.0764551162719727,
      "eval_rewards/margins": 2.736663341522217,
      "eval_rewards/rejected": -0.6602078676223755,
      "eval_runtime": 8.135,
      "eval_samples_per_second": 3.442,
      "eval_steps_per_second": 1.721,
      "step": 300
    },
    {
      "epoch": 5.04,
      "grad_norm": 3.4099764823913574,
      "learning_rate": 8.309859154929578e-07,
      "logits/chosen": -1.8781278133392334,
      "logits/rejected": -1.8944752216339111,
      "logps/chosen": -38.76420593261719,
      "logps/rejected": -69.38748931884766,
      "loss": 0.4198,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 1.978071928024292,
      "rewards/margins": 2.7254767417907715,
      "rewards/rejected": -0.7474048137664795,
      "step": 310
    },
    {
      "epoch": 5.2,
      "grad_norm": 0.0851665586233139,
      "learning_rate": 6.901408450704226e-07,
      "logits/chosen": -1.8191150426864624,
      "logits/rejected": -1.8379017114639282,
      "logps/chosen": -37.87184524536133,
      "logps/rejected": -69.2011947631836,
      "loss": 0.4166,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 2.0732247829437256,
      "rewards/margins": 2.8314859867095947,
      "rewards/rejected": -0.7582610845565796,
      "step": 320
    },
    {
      "epoch": 5.37,
      "grad_norm": 0.01842368021607399,
      "learning_rate": 5.492957746478875e-07,
      "logits/chosen": -1.8699190616607666,
      "logits/rejected": -1.8912782669067383,
      "logps/chosen": -34.895206451416016,
      "logps/rejected": -63.4892463684082,
      "loss": 0.4359,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 1.559017300605774,
      "rewards/margins": 2.468773126602173,
      "rewards/rejected": -0.9097555875778198,
      "step": 330
    },
    {
      "epoch": 5.37,
      "eval_logits/chosen": -1.7724134922027588,
      "eval_logits/rejected": -1.8115413188934326,
      "eval_logps/chosen": -44.29490280151367,
      "eval_logps/rejected": -85.83031463623047,
      "eval_loss": 0.394061416387558,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 2.1068155765533447,
      "eval_rewards/margins": 2.801534414291382,
      "eval_rewards/rejected": -0.6947189569473267,
      "eval_runtime": 8.1309,
      "eval_samples_per_second": 3.444,
      "eval_steps_per_second": 1.722,
      "step": 330
    },
    {
      "epoch": 5.53,
      "grad_norm": 0.5016012787818909,
      "learning_rate": 4.0845070422535216e-07,
      "logits/chosen": -1.8920612335205078,
      "logits/rejected": -1.9220449924468994,
      "logps/chosen": -36.482994079589844,
      "logps/rejected": -80.28797149658203,
      "loss": 0.4185,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 2.296034574508667,
      "rewards/margins": 3.2091434001922607,
      "rewards/rejected": -0.9131087064743042,
      "step": 340
    },
    {
      "epoch": 5.69,
      "grad_norm": 0.9488142728805542,
      "learning_rate": 2.676056338028169e-07,
      "logits/chosen": -1.808159589767456,
      "logits/rejected": -1.8219493627548218,
      "logps/chosen": -28.93426513671875,
      "logps/rejected": -51.149330139160156,
      "loss": 0.4757,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 1.7832109928131104,
      "rewards/margins": 2.264533758163452,
      "rewards/rejected": -0.4813228249549866,
      "step": 350
    },
    {
      "epoch": 5.85,
      "grad_norm": 0.636468231678009,
      "learning_rate": 1.267605633802817e-07,
      "logits/chosen": -1.8369605541229248,
      "logits/rejected": -1.8785759210586548,
      "logps/chosen": -39.411865234375,
      "logps/rejected": -105.97831726074219,
      "loss": 0.4413,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 2.0761430263519287,
      "rewards/margins": 2.9189963340759277,
      "rewards/rejected": -0.8428531885147095,
      "step": 360
    },
    {
      "epoch": 5.85,
      "eval_logits/chosen": -1.773056983947754,
      "eval_logits/rejected": -1.8121978044509888,
      "eval_logps/chosen": -44.24784469604492,
      "eval_logps/rejected": -85.88829040527344,
      "eval_loss": 0.3940436840057373,
      "eval_rewards/accuracies": 0.4642857015132904,
      "eval_rewards/chosen": 2.12093186378479,
      "eval_rewards/margins": 2.8330442905426025,
      "eval_rewards/rejected": -0.7121123671531677,
      "eval_runtime": 8.1312,
      "eval_samples_per_second": 3.444,
      "eval_steps_per_second": 1.722,
      "step": 360
    }
  ],
  "logging_steps": 10,
  "max_steps": 366,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 6,
  "save_steps": 90,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}