{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 108.05298614501953,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": 0.27033674716949463,
      "log_odds_ratio": -0.6787606477737427,
      "logits/chosen": -2.960299253463745,
      "logits/rejected": -2.877185106277466,
      "logps/chosen": -1.1196008920669556,
      "logps/rejected": -1.324358582496643,
      "loss": 47.1436,
      "nll_loss": 1.4213258028030396,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.22392019629478455,
      "rewards/margins": 0.04095155745744705,
      "rewards/rejected": -0.2648717164993286,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 60.31058883666992,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": 0.1787528693675995,
      "log_odds_ratio": -0.6780639886856079,
      "logits/chosen": -2.965952157974243,
      "logits/rejected": -2.846004009246826,
      "logps/chosen": -0.9941836595535278,
      "logps/rejected": -1.1146419048309326,
      "loss": 41.0236,
      "nll_loss": 1.2662255764007568,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.19883672893047333,
      "rewards/margins": 0.02409166470170021,
      "rewards/rejected": -0.22292840480804443,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 64.69369506835938,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": 0.1927437037229538,
      "log_odds_ratio": -0.6706465482711792,
      "logits/chosen": -2.8047168254852295,
      "logits/rejected": -2.7224316596984863,
      "logps/chosen": -0.7822418212890625,
      "logps/rejected": -0.890445351600647,
      "loss": 38.9336,
      "nll_loss": 1.2353577613830566,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1564483642578125,
      "rewards/margins": 0.021640703082084656,
      "rewards/rejected": -0.17808905243873596,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 47.254459381103516,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": 0.1654689460992813,
      "log_odds_ratio": -0.6847632527351379,
      "logits/chosen": -2.812252998352051,
      "logits/rejected": -2.759857654571533,
      "logps/chosen": -0.806632399559021,
      "logps/rejected": -0.9170172810554504,
      "loss": 36.5821,
      "nll_loss": 1.121074914932251,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.16132649779319763,
      "rewards/margins": 0.02207697369158268,
      "rewards/rejected": -0.18340346217155457,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 32.87034606933594,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": 0.11796001344919205,
      "log_odds_ratio": -0.7380820512771606,
      "logits/chosen": -2.8849737644195557,
      "logits/rejected": -2.837324857711792,
      "logps/chosen": -0.796675443649292,
      "logps/rejected": -0.8797528147697449,
      "loss": 35.6029,
      "nll_loss": 1.143587350845337,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.15933510661125183,
      "rewards/margins": 0.0166154894977808,
      "rewards/rejected": -0.17595058679580688,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 44.035709381103516,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 0.11726967990398407,
      "log_odds_ratio": -0.7224973440170288,
      "logits/chosen": -2.8627395629882812,
      "logits/rejected": -2.7379047870635986,
      "logps/chosen": -0.7604724764823914,
      "logps/rejected": -0.8269100189208984,
      "loss": 34.6175,
      "nll_loss": 1.0858324766159058,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1520944982767105,
      "rewards/margins": 0.013287502340972424,
      "rewards/rejected": -0.1653819978237152,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 35.993125915527344,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.11861761659383774,
      "log_odds_ratio": -0.7235144972801208,
      "logits/chosen": -2.8734326362609863,
      "logits/rejected": -2.924297571182251,
      "logps/chosen": -0.7806745171546936,
      "logps/rejected": -0.8397527933120728,
      "loss": 33.9132,
      "nll_loss": 1.096482515335083,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.15613490343093872,
      "rewards/margins": 0.01181565411388874,
      "rewards/rejected": -0.1679505705833435,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 31.309890747070312,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.1591743528842926,
      "log_odds_ratio": -0.6983242034912109,
      "logits/chosen": -2.8734216690063477,
      "logits/rejected": -2.8256828784942627,
      "logps/chosen": -0.7766333818435669,
      "logps/rejected": -0.8613529205322266,
      "loss": 34.0145,
      "nll_loss": 1.0602055788040161,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1553266942501068,
      "rewards/margins": 0.016943905502557755,
      "rewards/rejected": -0.17227059602737427,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 26.746047973632812,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.147725909948349,
      "log_odds_ratio": -0.7261999249458313,
      "logits/chosen": -2.816972017288208,
      "logits/rejected": -2.8251900672912598,
      "logps/chosen": -0.7309020161628723,
      "logps/rejected": -0.8109971880912781,
      "loss": 33.7617,
      "nll_loss": 1.0613343715667725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1461803913116455,
      "rewards/margins": 0.016019051894545555,
      "rewards/rejected": -0.1621994525194168,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 31.442655563354492,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.11614716053009033,
      "log_odds_ratio": -0.7367695569992065,
      "logits/chosen": -2.8157620429992676,
      "logits/rejected": -2.8260910511016846,
      "logps/chosen": -0.7965345978736877,
      "logps/rejected": -0.8452186584472656,
      "loss": 35.2613,
      "nll_loss": 1.1184347867965698,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.15930691361427307,
      "rewards/margins": 0.009736819192767143,
      "rewards/rejected": -0.16904373466968536,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 25.172645568847656,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 0.09427499771118164,
      "log_odds_ratio": -0.7385081052780151,
      "logits/chosen": -2.794956684112549,
      "logits/rejected": -2.7535877227783203,
      "logps/chosen": -0.7553600072860718,
      "logps/rejected": -0.8219752311706543,
      "loss": 32.9776,
      "nll_loss": 1.0361286401748657,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.15107201039791107,
      "rewards/margins": 0.013323022052645683,
      "rewards/rejected": -0.1643950492143631,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 27.215576171875,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 0.17814065515995026,
      "log_odds_ratio": -0.6895138025283813,
      "logits/chosen": -2.9259557723999023,
      "logits/rejected": -2.827220916748047,
      "logps/chosen": -0.7460590600967407,
      "logps/rejected": -0.828277587890625,
      "loss": 32.1892,
      "nll_loss": 1.0066068172454834,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.1492118090391159,
      "rewards/margins": 0.016443701460957527,
      "rewards/rejected": -0.16565552353858948,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 25.96778106689453,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 0.14030879735946655,
      "log_odds_ratio": -0.7043845057487488,
      "logits/chosen": -2.836535692214966,
      "logits/rejected": -2.8294970989227295,
      "logps/chosen": -0.7776893377304077,
      "logps/rejected": -0.8747210502624512,
      "loss": 32.3069,
      "nll_loss": 1.0067697763442993,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.15553787350654602,
      "rewards/margins": 0.019406333565711975,
      "rewards/rejected": -0.174944207072258,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 25.73870849609375,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 0.10110124200582504,
      "log_odds_ratio": -0.7182499170303345,
      "logits/chosen": -2.9134488105773926,
      "logits/rejected": -2.7823774814605713,
      "logps/chosen": -0.7712094187736511,
      "logps/rejected": -0.8390757441520691,
      "loss": 31.6312,
      "nll_loss": 1.0089954137802124,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1542418897151947,
      "rewards/margins": 0.013573260977864265,
      "rewards/rejected": -0.167815163731575,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 27.71648597717285,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 0.32593604922294617,
      "log_odds_ratio": -0.6149824857711792,
      "logits/chosen": -2.844346284866333,
      "logits/rejected": -2.8723151683807373,
      "logps/chosen": -0.6918070316314697,
      "logps/rejected": -0.8593597412109375,
      "loss": 32.1098,
      "nll_loss": 1.009124755859375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13836140930652618,
      "rewards/margins": 0.03351055458188057,
      "rewards/rejected": -0.17187196016311646,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 25.093658447265625,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 0.1939249038696289,
      "log_odds_ratio": -0.686955451965332,
      "logits/chosen": -2.961365222930908,
      "logits/rejected": -2.869180917739868,
      "logps/chosen": -0.7829879522323608,
      "logps/rejected": -0.8784705400466919,
      "loss": 31.996,
      "nll_loss": 1.0403110980987549,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1565975844860077,
      "rewards/margins": 0.019096530973911285,
      "rewards/rejected": -0.17569413781166077,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 21.968870162963867,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 0.15405571460723877,
      "log_odds_ratio": -0.691379189491272,
      "logits/chosen": -2.946697473526001,
      "logits/rejected": -2.7557473182678223,
      "logps/chosen": -0.7400744557380676,
      "logps/rejected": -0.8276380300521851,
      "loss": 31.9056,
      "nll_loss": 1.0012738704681396,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.14801490306854248,
      "rewards/margins": 0.017512720078229904,
      "rewards/rejected": -0.16552762687206268,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 24.323413848876953,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 0.2325698584318161,
      "log_odds_ratio": -0.6755145788192749,
      "logits/chosen": -2.9155423641204834,
      "logits/rejected": -2.8289742469787598,
      "logps/chosen": -0.7051252126693726,
      "logps/rejected": -0.8203322291374207,
      "loss": 32.5934,
      "nll_loss": 0.9984562993049622,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.14102503657341003,
      "rewards/margins": 0.023041393607854843,
      "rewards/rejected": -0.16406643390655518,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 23.40336036682129,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 0.25212088227272034,
      "log_odds_ratio": -0.6699780225753784,
      "logits/chosen": -2.8937487602233887,
      "logits/rejected": -2.7795820236206055,
      "logps/chosen": -0.6867400407791138,
      "logps/rejected": -0.8178392648696899,
      "loss": 32.0021,
      "nll_loss": 0.9535757899284363,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1373480260372162,
      "rewards/margins": 0.026219844818115234,
      "rewards/rejected": -0.16356787085533142,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 23.989849090576172,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 0.05919067934155464,
      "log_odds_ratio": -0.7570820450782776,
      "logits/chosen": -2.867305040359497,
      "logits/rejected": -2.7521088123321533,
      "logps/chosen": -0.7711822986602783,
      "logps/rejected": -0.8085872530937195,
      "loss": 32.0116,
      "nll_loss": 1.0364195108413696,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.15423646569252014,
      "rewards/margins": 0.007480976637452841,
      "rewards/rejected": -0.16171741485595703,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 23.476768493652344,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 0.15499885380268097,
      "log_odds_ratio": -0.7011421918869019,
      "logits/chosen": -2.934399127960205,
      "logits/rejected": -2.7926974296569824,
      "logps/chosen": -0.7334684729576111,
      "logps/rejected": -0.8502451181411743,
      "loss": 31.8454,
      "nll_loss": 0.992477297782898,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.14669370651245117,
      "rewards/margins": 0.02335532382130623,
      "rewards/rejected": -0.1700490266084671,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 24.518178939819336,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 0.48953691124916077,
      "log_odds_ratio": -0.5804011225700378,
      "logits/chosen": -2.850722074508667,
      "logits/rejected": -2.777690887451172,
      "logps/chosen": -0.5838567614555359,
      "logps/rejected": -0.8051884174346924,
      "loss": 24.0268,
      "nll_loss": 0.8107532262802124,
      "rewards/accuracies": 0.7062937021255493,
      "rewards/chosen": -0.11677134782075882,
      "rewards/margins": 0.04426633194088936,
      "rewards/rejected": -0.16103769838809967,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 27.153846740722656,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 0.6579824686050415,
      "log_odds_ratio": -0.5037034749984741,
      "logits/chosen": -2.88224458694458,
      "logits/rejected": -2.683182954788208,
      "logps/chosen": -0.5709269046783447,
      "logps/rejected": -0.8794952630996704,
      "loss": 25.9524,
      "nll_loss": 0.7917619347572327,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.1141853779554367,
      "rewards/margins": 0.06171366572380066,
      "rewards/rejected": -0.17589905858039856,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 24.396657943725586,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 0.7225304841995239,
      "log_odds_ratio": -0.49574628472328186,
      "logits/chosen": -2.8413453102111816,
      "logits/rejected": -2.783343553543091,
      "logps/chosen": -0.5242325067520142,
      "logps/rejected": -0.8295241594314575,
      "loss": 25.8745,
      "nll_loss": 0.786653995513916,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1048465147614479,
      "rewards/margins": 0.06105831265449524,
      "rewards/rejected": -0.16590481996536255,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 25.048795700073242,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 0.7397602796554565,
      "log_odds_ratio": -0.4922989308834076,
      "logits/chosen": -2.8375182151794434,
      "logits/rejected": -2.7994544506073,
      "logps/chosen": -0.5756106972694397,
      "logps/rejected": -0.9090244174003601,
      "loss": 24.2883,
      "nll_loss": 0.7707120180130005,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.11512213945388794,
      "rewards/margins": 0.06668277084827423,
      "rewards/rejected": -0.18180491030216217,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 24.017641067504883,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 0.619869589805603,
      "log_odds_ratio": -0.5279988050460815,
      "logits/chosen": -2.8795347213745117,
      "logits/rejected": -2.7905712127685547,
      "logps/chosen": -0.5735269784927368,
      "logps/rejected": -0.8608913421630859,
      "loss": 24.6652,
      "nll_loss": 0.7880049347877502,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.11470538377761841,
      "rewards/margins": 0.057472895830869675,
      "rewards/rejected": -0.17217828333377838,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 23.851089477539062,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 0.685693085193634,
      "log_odds_ratio": -0.5290096998214722,
      "logits/chosen": -2.858466625213623,
      "logits/rejected": -2.7995998859405518,
      "logps/chosen": -0.5094814896583557,
      "logps/rejected": -0.7905184030532837,
      "loss": 24.8932,
      "nll_loss": 0.7591756582260132,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10189628601074219,
      "rewards/margins": 0.05620738863945007,
      "rewards/rejected": -0.15810367465019226,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 25.953433990478516,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 0.6586907505989075,
      "log_odds_ratio": -0.5338261723518372,
      "logits/chosen": -2.802279233932495,
      "logits/rejected": -2.8112754821777344,
      "logps/chosen": -0.6032203435897827,
      "logps/rejected": -0.907819926738739,
      "loss": 26.2333,
      "nll_loss": 0.8216356039047241,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.12064407765865326,
      "rewards/margins": 0.06091991066932678,
      "rewards/rejected": -0.18156400322914124,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 25.108062744140625,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 0.5383874177932739,
      "log_odds_ratio": -0.5537405610084534,
      "logits/chosen": -2.90958833694458,
      "logits/rejected": -2.7673959732055664,
      "logps/chosen": -0.6065815687179565,
      "logps/rejected": -0.8782438039779663,
      "loss": 26.0014,
      "nll_loss": 0.8171162605285645,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.1213163286447525,
      "rewards/margins": 0.054332435131073,
      "rewards/rejected": -0.1756487637758255,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 26.676942825317383,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 0.5983611345291138,
      "log_odds_ratio": -0.547215461730957,
      "logits/chosen": -2.847853422164917,
      "logits/rejected": -2.8667774200439453,
      "logps/chosen": -0.563664436340332,
      "logps/rejected": -0.8132143020629883,
      "loss": 25.1239,
      "nll_loss": 0.7761403918266296,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11273287236690521,
      "rewards/margins": 0.049909986555576324,
      "rewards/rejected": -0.16264286637306213,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 25.11577606201172,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 0.674574077129364,
      "log_odds_ratio": -0.5198480486869812,
      "logits/chosen": -2.9048805236816406,
      "logits/rejected": -2.7696218490600586,
      "logps/chosen": -0.5536966919898987,
      "logps/rejected": -0.8423013687133789,
      "loss": 25.521,
      "nll_loss": 0.8120133280754089,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11073935031890869,
      "rewards/margins": 0.05772091820836067,
      "rewards/rejected": -0.16846027970314026,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 25.947433471679688,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 0.5704504251480103,
      "log_odds_ratio": -0.5273481607437134,
      "logits/chosen": -2.8649063110351562,
      "logits/rejected": -2.8651492595672607,
      "logps/chosen": -0.6272570490837097,
      "logps/rejected": -0.9066923260688782,
      "loss": 26.2994,
      "nll_loss": 0.8410785794258118,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.12545141577720642,
      "rewards/margins": 0.05588705092668533,
      "rewards/rejected": -0.18133845925331116,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 24.40997886657715,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 0.8175321817398071,
      "log_odds_ratio": -0.46456480026245117,
      "logits/chosen": -2.8088977336883545,
      "logits/rejected": -2.8634910583496094,
      "logps/chosen": -0.5574014782905579,
      "logps/rejected": -0.9002832174301147,
      "loss": 25.9286,
      "nll_loss": 0.7942954301834106,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.11148029565811157,
      "rewards/margins": 0.06857635080814362,
      "rewards/rejected": -0.1800566464662552,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 26.33037757873535,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 0.7566910982131958,
      "log_odds_ratio": -0.477250874042511,
      "logits/chosen": -2.8108813762664795,
      "logits/rejected": -2.784144878387451,
      "logps/chosen": -0.5413329601287842,
      "logps/rejected": -0.874762237071991,
      "loss": 25.1232,
      "nll_loss": 0.7829151153564453,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.10826659202575684,
      "rewards/margins": 0.06668586283922195,
      "rewards/rejected": -0.1749524623155594,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 25.30082130432129,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 0.6260709762573242,
      "log_odds_ratio": -0.5326700210571289,
      "logits/chosen": -2.8064045906066895,
      "logits/rejected": -2.7094810009002686,
      "logps/chosen": -0.5634613633155823,
      "logps/rejected": -0.8632895350456238,
      "loss": 25.126,
      "nll_loss": 0.8014172315597534,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11269225925207138,
      "rewards/margins": 0.05996562913060188,
      "rewards/rejected": -0.17265789210796356,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 25.368301391601562,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 0.49957117438316345,
      "log_odds_ratio": -0.5909265279769897,
      "logits/chosen": -2.728269338607788,
      "logits/rejected": -2.769505739212036,
      "logps/chosen": -0.5491189360618591,
      "logps/rejected": -0.7592207193374634,
      "loss": 25.3579,
      "nll_loss": 0.7878440618515015,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1098237857222557,
      "rewards/margins": 0.04202036187052727,
      "rewards/rejected": -0.15184415876865387,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 24.203075408935547,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 0.6133986115455627,
      "log_odds_ratio": -0.5178900957107544,
      "logits/chosen": -2.849193572998047,
      "logits/rejected": -2.8242928981781006,
      "logps/chosen": -0.5669192671775818,
      "logps/rejected": -0.837879478931427,
      "loss": 25.8406,
      "nll_loss": 0.8041340112686157,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.11338386684656143,
      "rewards/margins": 0.054192036390304565,
      "rewards/rejected": -0.1675758957862854,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 23.888410568237305,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 0.7135507464408875,
      "log_odds_ratio": -0.49830254912376404,
      "logits/chosen": -2.871323585510254,
      "logits/rejected": -2.716801404953003,
      "logps/chosen": -0.5923060774803162,
      "logps/rejected": -0.9126560091972351,
      "loss": 25.5896,
      "nll_loss": 0.7930334806442261,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.1184612289071083,
      "rewards/margins": 0.0640699714422226,
      "rewards/rejected": -0.1825312077999115,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 26.604984283447266,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 0.7206512689590454,
      "log_odds_ratio": -0.5070890784263611,
      "logits/chosen": -2.8227410316467285,
      "logits/rejected": -2.683023452758789,
      "logps/chosen": -0.5759368538856506,
      "logps/rejected": -0.8958457112312317,
      "loss": 25.1795,
      "nll_loss": 0.7880135774612427,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11518736928701401,
      "rewards/margins": 0.06398177146911621,
      "rewards/rejected": -0.17916914820671082,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 24.46125602722168,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 0.6902771592140198,
      "log_odds_ratio": -0.5180584788322449,
      "logits/chosen": -2.8943402767181396,
      "logits/rejected": -2.8360090255737305,
      "logps/chosen": -0.574259877204895,
      "logps/rejected": -0.8920720219612122,
      "loss": 26.2124,
      "nll_loss": 0.7853151559829712,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11485197395086288,
      "rewards/margins": 0.06356243789196014,
      "rewards/rejected": -0.17841440439224243,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 22.397045135498047,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 0.5525709986686707,
      "log_odds_ratio": -0.5616833567619324,
      "logits/chosen": -2.8779659271240234,
      "logits/rejected": -2.7355799674987793,
      "logps/chosen": -0.5917361378669739,
      "logps/rejected": -0.8651636242866516,
      "loss": 25.1435,
      "nll_loss": 0.7885797619819641,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11834722757339478,
      "rewards/margins": 0.054685525596141815,
      "rewards/rejected": -0.173032745718956,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 23.24612808227539,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 0.6066923141479492,
      "log_odds_ratio": -0.5472615361213684,
      "logits/chosen": -2.8647236824035645,
      "logits/rejected": -2.9078686237335205,
      "logps/chosen": -0.5761645436286926,
      "logps/rejected": -0.8358944058418274,
      "loss": 23.8296,
      "nll_loss": 0.7535713911056519,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1152329072356224,
      "rewards/margins": 0.051945965737104416,
      "rewards/rejected": -0.16717885434627533,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 25.06052017211914,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 1.0616474151611328,
      "log_odds_ratio": -0.4191078841686249,
      "logits/chosen": -2.7932209968566895,
      "logits/rejected": -2.7296035289764404,
      "logps/chosen": -0.471683144569397,
      "logps/rejected": -0.8782158493995667,
      "loss": 19.6408,
      "nll_loss": 0.6726168990135193,
      "rewards/accuracies": 0.811188817024231,
      "rewards/chosen": -0.0943366289138794,
      "rewards/margins": 0.08130654692649841,
      "rewards/rejected": -0.1756431758403778,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 31.775066375732422,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 1.2414283752441406,
      "log_odds_ratio": -0.37072741985321045,
      "logits/chosen": -2.8022265434265137,
      "logits/rejected": -2.7584967613220215,
      "logps/chosen": -0.4369787573814392,
      "logps/rejected": -0.9211339950561523,
      "loss": 19.9908,
      "nll_loss": 0.647934079170227,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.08739574998617172,
      "rewards/margins": 0.09683104604482651,
      "rewards/rejected": -0.18422682583332062,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 26.847965240478516,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 1.1905401945114136,
      "log_odds_ratio": -0.3879564702510834,
      "logits/chosen": -2.7660067081451416,
      "logits/rejected": -2.8042025566101074,
      "logps/chosen": -0.4038045406341553,
      "logps/rejected": -0.8510030508041382,
      "loss": 20.6864,
      "nll_loss": 0.6349369287490845,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0807609036564827,
      "rewards/margins": 0.08943970501422882,
      "rewards/rejected": -0.17020061612129211,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 26.880268096923828,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 1.3126904964447021,
      "log_odds_ratio": -0.3657984137535095,
      "logits/chosen": -2.776597023010254,
      "logits/rejected": -2.7055163383483887,
      "logps/chosen": -0.41880035400390625,
      "logps/rejected": -0.8985490798950195,
      "loss": 19.3404,
      "nll_loss": 0.5868525505065918,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.08376006782054901,
      "rewards/margins": 0.09594975411891937,
      "rewards/rejected": -0.17970982193946838,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 31.306821823120117,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 1.1681408882141113,
      "log_odds_ratio": -0.4105006754398346,
      "logits/chosen": -2.784541368484497,
      "logits/rejected": -2.7821319103240967,
      "logps/chosen": -0.44898858666419983,
      "logps/rejected": -0.870564341545105,
      "loss": 20.274,
      "nll_loss": 0.6389156579971313,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.08979771286249161,
      "rewards/margins": 0.08431516587734222,
      "rewards/rejected": -0.17411288619041443,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 26.963720321655273,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 1.2784669399261475,
      "log_odds_ratio": -0.3686579167842865,
      "logits/chosen": -2.7864890098571777,
      "logits/rejected": -2.8318161964416504,
      "logps/chosen": -0.43692564964294434,
      "logps/rejected": -0.9061982035636902,
      "loss": 20.5384,
      "nll_loss": 0.659681499004364,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0873851329088211,
      "rewards/margins": 0.09385450184345245,
      "rewards/rejected": -0.18123963475227356,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 27.23887062072754,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 1.3585851192474365,
      "log_odds_ratio": -0.3239150047302246,
      "logits/chosen": -2.794093370437622,
      "logits/rejected": -2.7042746543884277,
      "logps/chosen": -0.41034674644470215,
      "logps/rejected": -0.9184103012084961,
      "loss": 19.5157,
      "nll_loss": 0.6223703026771545,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.08206935226917267,
      "rewards/margins": 0.10161270946264267,
      "rewards/rejected": -0.18368205428123474,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 27.25604248046875,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 1.3011986017227173,
      "log_odds_ratio": -0.3395061194896698,
      "logits/chosen": -2.805485248565674,
      "logits/rejected": -2.7763752937316895,
      "logps/chosen": -0.44493547081947327,
      "logps/rejected": -0.9599026441574097,
      "loss": 19.1928,
      "nll_loss": 0.6162556409835815,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.08898709714412689,
      "rewards/margins": 0.102993443608284,
      "rewards/rejected": -0.1919805258512497,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 27.4134578704834,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 1.4432458877563477,
      "log_odds_ratio": -0.32654649019241333,
      "logits/chosen": -2.748802661895752,
      "logits/rejected": -2.7334983348846436,
      "logps/chosen": -0.3842257261276245,
      "logps/rejected": -0.9072483777999878,
      "loss": 19.1784,
      "nll_loss": 0.5993472337722778,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.07684514671564102,
      "rewards/margins": 0.1046045571565628,
      "rewards/rejected": -0.18144968152046204,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 28.401473999023438,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 1.488121747970581,
      "log_odds_ratio": -0.3242756724357605,
      "logits/chosen": -2.7776598930358887,
      "logits/rejected": -2.6771626472473145,
      "logps/chosen": -0.392120897769928,
      "logps/rejected": -0.9110913276672363,
      "loss": 19.3257,
      "nll_loss": 0.5952539443969727,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.07842419296503067,
      "rewards/margins": 0.10379407554864883,
      "rewards/rejected": -0.1822182834148407,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 28.88060760498047,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 1.3083126544952393,
      "log_odds_ratio": -0.3742141127586365,
      "logits/chosen": -2.752208709716797,
      "logits/rejected": -2.7710514068603516,
      "logps/chosen": -0.4226323068141937,
      "logps/rejected": -0.9085744619369507,
      "loss": 19.018,
      "nll_loss": 0.5947185158729553,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.08452647179365158,
      "rewards/margins": 0.09718842804431915,
      "rewards/rejected": -0.18171489238739014,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 26.92013168334961,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 1.292988657951355,
      "log_odds_ratio": -0.3492991328239441,
      "logits/chosen": -2.8405444622039795,
      "logits/rejected": -2.795259475708008,
      "logps/chosen": -0.4322505593299866,
      "logps/rejected": -0.9027539491653442,
      "loss": 19.1197,
      "nll_loss": 0.6248734593391418,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08645011484622955,
      "rewards/margins": 0.09410068392753601,
      "rewards/rejected": -0.18055078387260437,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 25.733835220336914,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 1.3847897052764893,
      "log_odds_ratio": -0.33749812841415405,
      "logits/chosen": -2.8116893768310547,
      "logits/rejected": -2.695439100265503,
      "logps/chosen": -0.4366413950920105,
      "logps/rejected": -0.971570611000061,
      "loss": 20.1018,
      "nll_loss": 0.6324763894081116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08732827007770538,
      "rewards/margins": 0.10698586702346802,
      "rewards/rejected": -0.1943141520023346,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 26.444095611572266,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 1.2408794164657593,
      "log_odds_ratio": -0.3706284165382385,
      "logits/chosen": -2.783261775970459,
      "logits/rejected": -2.787781238555908,
      "logps/chosen": -0.4150269031524658,
      "logps/rejected": -0.8832289576530457,
      "loss": 19.2227,
      "nll_loss": 0.5912734270095825,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.0830053836107254,
      "rewards/margins": 0.09364042431116104,
      "rewards/rejected": -0.17664580047130585,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 29.028366088867188,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 1.2308132648468018,
      "log_odds_ratio": -0.3709443509578705,
      "logits/chosen": -2.829824924468994,
      "logits/rejected": -2.750471830368042,
      "logps/chosen": -0.4345945417881012,
      "logps/rejected": -0.8928596377372742,
      "loss": 19.7763,
      "nll_loss": 0.6206212639808655,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.086918905377388,
      "rewards/margins": 0.09165303409099579,
      "rewards/rejected": -0.17857195436954498,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 27.85979652404785,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 1.3631649017333984,
      "log_odds_ratio": -0.34059271216392517,
      "logits/chosen": -2.741471529006958,
      "logits/rejected": -2.7575411796569824,
      "logps/chosen": -0.3912351727485657,
      "logps/rejected": -0.8874975442886353,
      "loss": 19.7779,
      "nll_loss": 0.6171984672546387,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.07824703305959702,
      "rewards/margins": 0.09925246983766556,
      "rewards/rejected": -0.17749951779842377,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 28.014738082885742,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 1.1799755096435547,
      "log_odds_ratio": -0.39853397011756897,
      "logits/chosen": -2.691950559616089,
      "logits/rejected": -2.7646536827087402,
      "logps/chosen": -0.45684152841567993,
      "logps/rejected": -0.9104090929031372,
      "loss": 19.6944,
      "nll_loss": 0.6349056959152222,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09136830270290375,
      "rewards/margins": 0.0907135158777237,
      "rewards/rejected": -0.18208181858062744,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 29.324554443359375,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 1.1676690578460693,
      "log_odds_ratio": -0.3855035603046417,
      "logits/chosen": -2.804230213165283,
      "logits/rejected": -2.6715846061706543,
      "logps/chosen": -0.44394516944885254,
      "logps/rejected": -0.9196891784667969,
      "loss": 19.6876,
      "nll_loss": 0.6331617832183838,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.08878904581069946,
      "rewards/margins": 0.09514881670475006,
      "rewards/rejected": -0.18393786251544952,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 26.675434112548828,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 1.2555936574935913,
      "log_odds_ratio": -0.3468943238258362,
      "logits/chosen": -2.866672992706299,
      "logits/rejected": -2.79132342338562,
      "logps/chosen": -0.4585191607475281,
      "logps/rejected": -0.9410701990127563,
      "loss": 19.4548,
      "nll_loss": 0.6043152809143066,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.09170383214950562,
      "rewards/margins": 0.09651019424200058,
      "rewards/rejected": -0.1882140338420868,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 30.77156639099121,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 1.2845065593719482,
      "log_odds_ratio": -0.3690781891345978,
      "logits/chosen": -2.767889976501465,
      "logits/rejected": -2.665135383605957,
      "logps/chosen": -0.41893452405929565,
      "logps/rejected": -0.8851953744888306,
      "loss": 19.3972,
      "nll_loss": 0.6142546534538269,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.08378690481185913,
      "rewards/margins": 0.09325218200683594,
      "rewards/rejected": -0.17703908681869507,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 27.507280349731445,
      "learning_rate": 0.0,
      "log_odds_chosen": 1.394679069519043,
      "log_odds_ratio": -0.32423505187034607,
      "logits/chosen": -2.744528293609619,
      "logits/rejected": -2.668449640274048,
      "logps/chosen": -0.4062735140323639,
      "logps/rejected": -0.9089689254760742,
      "loss": 18.6834,
      "nll_loss": 0.5965942144393921,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.08125469833612442,
      "rewards/margins": 0.10053907334804535,
      "rewards/rejected": -0.18179377913475037,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 26.480164833674355,
      "train_runtime": 6938.2073,
      "train_samples_per_second": 2.919,
      "train_steps_per_second": 0.045
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}