| { | |
| "best_metric": 1.265723466873169, | |
| "best_model_checkpoint": "saves/Gemma-7B-It/lora/orpo-salt/checkpoint-1500", | |
| "epoch": 2.9969690846635686, | |
| "eval_steps": 500, | |
| "global_step": 1854, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01616488179430188, | |
| "grad_norm": 4.377878189086914, | |
| "learning_rate": 4.999648198770648e-06, | |
| "logits/chosen": 209.9345245361328, | |
| "logits/rejected": 210.6967315673828, | |
| "logps/chosen": -2.4765946865081787, | |
| "logps/rejected": -2.9186055660247803, | |
| "loss": 2.5449, | |
| "odds_ratio_loss": 0.6828715205192566, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.24765947461128235, | |
| "rewards/margins": 0.04420109838247299, | |
| "rewards/rejected": -0.29186058044433594, | |
| "sft_loss": 2.4765946865081787, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03232976358860376, | |
| "grad_norm": 2.781564950942993, | |
| "learning_rate": 4.998578646361359e-06, | |
| "logits/chosen": 210.4038543701172, | |
| "logits/rejected": 212.20718383789062, | |
| "logps/chosen": -2.4702863693237305, | |
| "logps/rejected": -2.504176616668701, | |
| "loss": 2.564, | |
| "odds_ratio_loss": 0.9375804662704468, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.2470286339521408, | |
| "rewards/margins": 0.0033890369813889265, | |
| "rewards/rejected": -0.25041764974594116, | |
| "sft_loss": 2.4702863693237305, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.04849464538290564, | |
| "grad_norm": 5.785957336425781, | |
| "learning_rate": 4.996791614004449e-06, | |
| "logits/chosen": 209.83865356445312, | |
| "logits/rejected": 212.08535766601562, | |
| "logps/chosen": -2.6004161834716797, | |
| "logps/rejected": -2.695502758026123, | |
| "loss": 2.6963, | |
| "odds_ratio_loss": 0.9585107564926147, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.26004162430763245, | |
| "rewards/margins": 0.009508667513728142, | |
| "rewards/rejected": -0.26955026388168335, | |
| "sft_loss": 2.6004161834716797, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.06465952717720752, | |
| "grad_norm": 7.009506702423096, | |
| "learning_rate": 4.994287614855618e-06, | |
| "logits/chosen": 210.0410614013672, | |
| "logits/rejected": 211.40286254882812, | |
| "logps/chosen": -2.6340386867523193, | |
| "logps/rejected": -2.6249070167541504, | |
| "loss": 2.7374, | |
| "odds_ratio_loss": 1.0338027477264404, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.26340389251708984, | |
| "rewards/margins": -0.0009131729602813721, | |
| "rewards/rejected": -0.2624906897544861, | |
| "sft_loss": 2.6340386867523193, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0808244089715094, | |
| "grad_norm": 4.594735145568848, | |
| "learning_rate": 4.991067367951343e-06, | |
| "logits/chosen": 219.71932983398438, | |
| "logits/rejected": 219.6745147705078, | |
| "logps/chosen": -2.3416378498077393, | |
| "logps/rejected": -2.4940619468688965, | |
| "loss": 2.4215, | |
| "odds_ratio_loss": 0.7983426451683044, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.2341637909412384, | |
| "rewards/margins": 0.015242427587509155, | |
| "rewards/rejected": -0.24940618872642517, | |
| "sft_loss": 2.3416378498077393, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.09698929076581128, | |
| "grad_norm": 2.953855276107788, | |
| "learning_rate": 4.987131798002389e-06, | |
| "logits/chosen": 217.3623504638672, | |
| "logits/rejected": 218.24862670898438, | |
| "logps/chosen": -2.2888264656066895, | |
| "logps/rejected": -2.6409952640533447, | |
| "loss": 2.3829, | |
| "odds_ratio_loss": 0.940882682800293, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.22888264060020447, | |
| "rewards/margins": 0.035216934978961945, | |
| "rewards/rejected": -0.2640995383262634, | |
| "sft_loss": 2.2888264656066895, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11315417256011315, | |
| "grad_norm": 4.224141597747803, | |
| "learning_rate": 4.982482035128285e-06, | |
| "logits/chosen": 217.9263458251953, | |
| "logits/rejected": 218.54122924804688, | |
| "logps/chosen": -2.326590061187744, | |
| "logps/rejected": -2.605003833770752, | |
| "loss": 2.4191, | |
| "odds_ratio_loss": 0.9254364967346191, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.2326590120792389, | |
| "rewards/margins": 0.027841363102197647, | |
| "rewards/rejected": -0.26050037145614624, | |
| "sft_loss": 2.326590061187744, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.12931905435441504, | |
| "grad_norm": 7.0222883224487305, | |
| "learning_rate": 4.9771194145328e-06, | |
| "logits/chosen": 224.885986328125, | |
| "logits/rejected": 225.7215576171875, | |
| "logps/chosen": -1.8678385019302368, | |
| "logps/rejected": -2.1334400177001953, | |
| "loss": 1.9429, | |
| "odds_ratio_loss": 0.7510749697685242, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.1867838352918625, | |
| "rewards/margins": 0.026560146361589432, | |
| "rewards/rejected": -0.213344007730484, | |
| "sft_loss": 1.8678385019302368, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.1454839361487169, | |
| "grad_norm": 9.777688026428223, | |
| "learning_rate": 4.971045476120532e-06, | |
| "logits/chosen": 226.64450073242188, | |
| "logits/rejected": 227.11874389648438, | |
| "logps/chosen": -1.9129263162612915, | |
| "logps/rejected": -2.10162091255188, | |
| "loss": 1.9975, | |
| "odds_ratio_loss": 0.8461491465568542, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.1912926286458969, | |
| "rewards/margins": 0.01886945590376854, | |
| "rewards/rejected": -0.21016211807727814, | |
| "sft_loss": 1.9129263162612915, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1616488179430188, | |
| "grad_norm": 3.441721200942993, | |
| "learning_rate": 4.964261964054713e-06, | |
| "logits/chosen": 230.32669067382812, | |
| "logits/rejected": 231.39498901367188, | |
| "logps/chosen": -1.8438594341278076, | |
| "logps/rejected": -2.1114680767059326, | |
| "loss": 1.923, | |
| "odds_ratio_loss": 0.7917153239250183, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.18438595533370972, | |
| "rewards/margins": 0.026760881766676903, | |
| "rewards/rejected": -0.21114683151245117, | |
| "sft_loss": 1.8438594341278076, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.17781369973732067, | |
| "grad_norm": 3.7387969493865967, | |
| "learning_rate": 4.956770826256372e-06, | |
| "logits/chosen": 233.9343719482422, | |
| "logits/rejected": 234.51516723632812, | |
| "logps/chosen": -1.6228179931640625, | |
| "logps/rejected": -1.8143441677093506, | |
| "loss": 1.6988, | |
| "odds_ratio_loss": 0.7598803043365479, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.1622818112373352, | |
| "rewards/margins": 0.01915261521935463, | |
| "rewards/rejected": -0.18143442273139954, | |
| "sft_loss": 1.6228179931640625, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.19397858153162256, | |
| "grad_norm": 2.157771110534668, | |
| "learning_rate": 4.94857421384497e-06, | |
| "logits/chosen": 235.01248168945312, | |
| "logits/rejected": 235.390869140625, | |
| "logps/chosen": -1.6021674871444702, | |
| "logps/rejected": -1.885866403579712, | |
| "loss": 1.6762, | |
| "odds_ratio_loss": 0.7404050230979919, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1602167785167694, | |
| "rewards/margins": 0.02836987003684044, | |
| "rewards/rejected": -0.18858662247657776, | |
| "sft_loss": 1.6021674871444702, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.21014346332592443, | |
| "grad_norm": 2.794867515563965, | |
| "learning_rate": 4.939674480520701e-06, | |
| "logits/chosen": 236.7910614013672, | |
| "logits/rejected": 237.41806030273438, | |
| "logps/chosen": -1.5707839727401733, | |
| "logps/rejected": -1.6964565515518188, | |
| "loss": 1.6508, | |
| "odds_ratio_loss": 0.8003607988357544, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.15707840025424957, | |
| "rewards/margins": 0.01256726123392582, | |
| "rewards/rejected": -0.16964565217494965, | |
| "sft_loss": 1.5707839727401733, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.2263083451202263, | |
| "grad_norm": 1.2237716913223267, | |
| "learning_rate": 4.930074181888613e-06, | |
| "logits/chosen": 240.5333251953125, | |
| "logits/rejected": 241.0712432861328, | |
| "logps/chosen": -1.6245830059051514, | |
| "logps/rejected": -1.83035409450531, | |
| "loss": 1.6912, | |
| "odds_ratio_loss": 0.6659940481185913, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.16245830059051514, | |
| "rewards/margins": 0.02057710848748684, | |
| "rewards/rejected": -0.18303541839122772, | |
| "sft_loss": 1.6245830059051514, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2424732269145282, | |
| "grad_norm": 3.366241693496704, | |
| "learning_rate": 4.91977607472475e-06, | |
| "logits/chosen": 240.38449096679688, | |
| "logits/rejected": 241.23794555664062, | |
| "logps/chosen": -1.5312227010726929, | |
| "logps/rejected": -1.6705970764160156, | |
| "loss": 1.6035, | |
| "odds_ratio_loss": 0.7224593162536621, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.15312227606773376, | |
| "rewards/margins": 0.013937436044216156, | |
| "rewards/rejected": -0.16705971956253052, | |
| "sft_loss": 1.5312227010726929, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2586381087088301, | |
| "grad_norm": 2.1750807762145996, | |
| "learning_rate": 4.908783116184534e-06, | |
| "logits/chosen": 240.67446899414062, | |
| "logits/rejected": 241.75424194335938, | |
| "logps/chosen": -1.4731028079986572, | |
| "logps/rejected": -1.7678340673446655, | |
| "loss": 1.5362, | |
| "odds_ratio_loss": 0.6307698488235474, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.1473102867603302, | |
| "rewards/margins": 0.029473140835762024, | |
| "rewards/rejected": -0.17678341269493103, | |
| "sft_loss": 1.4731028079986572, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.27480299050313195, | |
| "grad_norm": 2.9354147911071777, | |
| "learning_rate": 4.897098462953598e-06, | |
| "logits/chosen": 243.85806274414062, | |
| "logits/rejected": 244.68405151367188, | |
| "logps/chosen": -1.3806183338165283, | |
| "logps/rejected": -1.7258154153823853, | |
| "loss": 1.4423, | |
| "odds_ratio_loss": 0.6170833706855774, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.13806185126304626, | |
| "rewards/margins": 0.03451969474554062, | |
| "rewards/rejected": -0.1725815385580063, | |
| "sft_loss": 1.3806183338165283, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.2909678722974338, | |
| "grad_norm": 1.4452638626098633, | |
| "learning_rate": 4.884725470341331e-06, | |
| "logits/chosen": 242.984619140625, | |
| "logits/rejected": 243.6776580810547, | |
| "logps/chosen": -1.2990996837615967, | |
| "logps/rejected": -1.616987943649292, | |
| "loss": 1.3597, | |
| "odds_ratio_loss": 0.6057690382003784, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.12990999221801758, | |
| "rewards/margins": 0.031788814812898636, | |
| "rewards/rejected": -0.16169880330562592, | |
| "sft_loss": 1.2990996837615967, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3071327540917357, | |
| "grad_norm": 4.690347194671631, | |
| "learning_rate": 4.871667691317377e-06, | |
| "logits/chosen": 244.59634399414062, | |
| "logits/rejected": 244.5352325439453, | |
| "logps/chosen": -1.4848819971084595, | |
| "logps/rejected": -1.573250412940979, | |
| "loss": 1.5639, | |
| "odds_ratio_loss": 0.7902374267578125, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.14848819375038147, | |
| "rewards/margins": 0.008836844936013222, | |
| "rewards/rejected": -0.15732502937316895, | |
| "sft_loss": 1.4848819971084595, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3232976358860376, | |
| "grad_norm": 7.527270317077637, | |
| "learning_rate": 4.857928875491392e-06, | |
| "logits/chosen": 243.60494995117188, | |
| "logits/rejected": 244.3643035888672, | |
| "logps/chosen": -1.3324997425079346, | |
| "logps/rejected": -1.5205867290496826, | |
| "loss": 1.402, | |
| "odds_ratio_loss": 0.6946145296096802, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.13324996829032898, | |
| "rewards/margins": 0.018808716908097267, | |
| "rewards/rejected": -0.1520586758852005, | |
| "sft_loss": 1.3324997425079346, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.33946251768033947, | |
| "grad_norm": 2.1978328227996826, | |
| "learning_rate": 4.843512968036314e-06, | |
| "logits/chosen": 244.3915557861328, | |
| "logits/rejected": 244.58700561523438, | |
| "logps/chosen": -1.3562281131744385, | |
| "logps/rejected": -1.4892576932907104, | |
| "loss": 1.4274, | |
| "odds_ratio_loss": 0.7121940851211548, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.1356228142976761, | |
| "rewards/margins": 0.01330297440290451, | |
| "rewards/rejected": -0.1489257663488388, | |
| "sft_loss": 1.3562281131744385, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.35562739947464134, | |
| "grad_norm": 6.31206750869751, | |
| "learning_rate": 4.828424108555486e-06, | |
| "logits/chosen": 246.1901092529297, | |
| "logits/rejected": 246.36703491210938, | |
| "logps/chosen": -1.5392124652862549, | |
| "logps/rejected": -1.7705978155136108, | |
| "loss": 1.6086, | |
| "odds_ratio_loss": 0.6943382024765015, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.1539212465286255, | |
| "rewards/margins": 0.023138541728258133, | |
| "rewards/rejected": -0.17705979943275452, | |
| "sft_loss": 1.5392124652862549, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3717922812689432, | |
| "grad_norm": 1.1257890462875366, | |
| "learning_rate": 4.812666629893957e-06, | |
| "logits/chosen": 246.37399291992188, | |
| "logits/rejected": 246.72891235351562, | |
| "logps/chosen": -1.3704453706741333, | |
| "logps/rejected": -1.4485595226287842, | |
| "loss": 1.4433, | |
| "odds_ratio_loss": 0.7287623882293701, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.1370445340871811, | |
| "rewards/margins": 0.007811415940523148, | |
| "rewards/rejected": -0.14485594630241394, | |
| "sft_loss": 1.3704453706741333, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.3879571630632451, | |
| "grad_norm": 1.9700157642364502, | |
| "learning_rate": 4.796245056894273e-06, | |
| "logits/chosen": 244.54165649414062, | |
| "logits/rejected": 244.89407348632812, | |
| "logps/chosen": -1.4429550170898438, | |
| "logps/rejected": -1.5743396282196045, | |
| "loss": 1.5184, | |
| "odds_ratio_loss": 0.7547486424446106, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.14429552853107452, | |
| "rewards/margins": 0.013138455338776112, | |
| "rewards/rejected": -0.15743397176265717, | |
| "sft_loss": 1.4429550170898438, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.404122044857547, | |
| "grad_norm": 1.5832947492599487, | |
| "learning_rate": 4.779164105097148e-06, | |
| "logits/chosen": 246.41659545898438, | |
| "logits/rejected": 246.4707489013672, | |
| "logps/chosen": -1.3124094009399414, | |
| "logps/rejected": -1.5739551782608032, | |
| "loss": 1.3768, | |
| "odds_ratio_loss": 0.6443756818771362, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.13124093413352966, | |
| "rewards/margins": 0.026154566556215286, | |
| "rewards/rejected": -0.15739551186561584, | |
| "sft_loss": 1.3124094009399414, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.42028692665184886, | |
| "grad_norm": 2.2224152088165283, | |
| "learning_rate": 4.761428679387373e-06, | |
| "logits/chosen": 247.0335235595703, | |
| "logits/rejected": 247.7626953125, | |
| "logps/chosen": -1.2735482454299927, | |
| "logps/rejected": -1.5084031820297241, | |
| "loss": 1.3358, | |
| "odds_ratio_loss": 0.6226388216018677, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.12735481560230255, | |
| "rewards/margins": 0.02348550595343113, | |
| "rewards/rejected": -0.15084032714366913, | |
| "sft_loss": 1.2735482454299927, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4364518084461507, | |
| "grad_norm": 2.0271799564361572, | |
| "learning_rate": 4.7430438725853515e-06, | |
| "logits/chosen": 247.60205078125, | |
| "logits/rejected": 247.61654663085938, | |
| "logps/chosen": -1.3570277690887451, | |
| "logps/rejected": -1.714133858680725, | |
| "loss": 1.4226, | |
| "odds_ratio_loss": 0.6552777290344238, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.13570277392864227, | |
| "rewards/margins": 0.035710614174604416, | |
| "rewards/rejected": -0.171413391828537, | |
| "sft_loss": 1.3570277690887451, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4526166902404526, | |
| "grad_norm": 2.142329216003418, | |
| "learning_rate": 4.724014963984669e-06, | |
| "logits/chosen": 248.28439331054688, | |
| "logits/rejected": 249.0177459716797, | |
| "logps/chosen": -1.3674625158309937, | |
| "logps/rejected": -1.6127933263778687, | |
| "loss": 1.435, | |
| "odds_ratio_loss": 0.6751004457473755, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13674625754356384, | |
| "rewards/margins": 0.02453308179974556, | |
| "rewards/rejected": -0.1612793505191803, | |
| "sft_loss": 1.3674625158309937, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4687815720347545, | |
| "grad_norm": 2.8357582092285156, | |
| "learning_rate": 4.704347417836116e-06, | |
| "logits/chosen": 247.2007598876953, | |
| "logits/rejected": 247.60107421875, | |
| "logps/chosen": -1.2728191614151, | |
| "logps/rejected": -1.5069888830184937, | |
| "loss": 1.3382, | |
| "odds_ratio_loss": 0.6542028784751892, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12728191912174225, | |
| "rewards/margins": 0.023416969925165176, | |
| "rewards/rejected": -0.15069888532161713, | |
| "sft_loss": 1.2728191614151, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.4849464538290564, | |
| "grad_norm": 3.075584888458252, | |
| "learning_rate": 4.684046881778603e-06, | |
| "logits/chosen": 247.69580078125, | |
| "logits/rejected": 247.7963409423828, | |
| "logps/chosen": -1.3267529010772705, | |
| "logps/rejected": -1.46425461769104, | |
| "loss": 1.3929, | |
| "odds_ratio_loss": 0.6614553332328796, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.13267529010772705, | |
| "rewards/margins": 0.013750175014138222, | |
| "rewards/rejected": -0.14642547070980072, | |
| "sft_loss": 1.3267529010772705, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5011113356233583, | |
| "grad_norm": 1.1745957136154175, | |
| "learning_rate": 4.663119185217409e-06, | |
| "logits/chosen": 247.5077667236328, | |
| "logits/rejected": 247.80752563476562, | |
| "logps/chosen": -1.2750051021575928, | |
| "logps/rejected": -1.5364891290664673, | |
| "loss": 1.3385, | |
| "odds_ratio_loss": 0.6352204084396362, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.1275005042552948, | |
| "rewards/margins": 0.026148397475481033, | |
| "rewards/rejected": -0.15364892780780792, | |
| "sft_loss": 1.2750051021575928, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5172762174176602, | |
| "grad_norm": 1.1816167831420898, | |
| "learning_rate": 4.641570337650232e-06, | |
| "logits/chosen": 248.5536651611328, | |
| "logits/rejected": 248.5113067626953, | |
| "logps/chosen": -1.1914936304092407, | |
| "logps/rejected": -1.4479808807373047, | |
| "loss": 1.2531, | |
| "odds_ratio_loss": 0.615585446357727, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.11914938688278198, | |
| "rewards/margins": 0.025648722425103188, | |
| "rewards/rejected": -0.14479808509349823, | |
| "sft_loss": 1.1914936304092407, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.533441099211962, | |
| "grad_norm": 6.805661678314209, | |
| "learning_rate": 4.61940652694154e-06, | |
| "logits/chosen": 246.8784637451172, | |
| "logits/rejected": 247.60842895507812, | |
| "logps/chosen": -1.371927261352539, | |
| "logps/rejected": -1.4951013326644897, | |
| "loss": 1.444, | |
| "odds_ratio_loss": 0.7208858728408813, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.1371927261352539, | |
| "rewards/margins": 0.012317392975091934, | |
| "rewards/rejected": -0.14951011538505554, | |
| "sft_loss": 1.371927261352539, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.5496059810062639, | |
| "grad_norm": 2.8288872241973877, | |
| "learning_rate": 4.596634117545689e-06, | |
| "logits/chosen": 248.96542358398438, | |
| "logits/rejected": 249.38369750976562, | |
| "logps/chosen": -1.3861172199249268, | |
| "logps/rejected": -1.6291033029556274, | |
| "loss": 1.4514, | |
| "odds_ratio_loss": 0.6529659032821655, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.13861171901226044, | |
| "rewards/margins": 0.02429860271513462, | |
| "rewards/rejected": -0.1629103422164917, | |
| "sft_loss": 1.3861172199249268, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5657708628005658, | |
| "grad_norm": 2.343557834625244, | |
| "learning_rate": 4.573259648679335e-06, | |
| "logits/chosen": 247.5604248046875, | |
| "logits/rejected": 247.8214111328125, | |
| "logps/chosen": -1.3334286212921143, | |
| "logps/rejected": -1.642163634300232, | |
| "loss": 1.3937, | |
| "odds_ratio_loss": 0.603044331073761, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.13334286212921143, | |
| "rewards/margins": 0.030873507261276245, | |
| "rewards/rejected": -0.16421635448932648, | |
| "sft_loss": 1.3334286212921143, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.5819357445948676, | |
| "grad_norm": 6.341250896453857, | |
| "learning_rate": 4.549289832443663e-06, | |
| "logits/chosen": 249.6760711669922, | |
| "logits/rejected": 249.2826385498047, | |
| "logps/chosen": -1.2829958200454712, | |
| "logps/rejected": -1.5420135259628296, | |
| "loss": 1.351, | |
| "odds_ratio_loss": 0.6805331110954285, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12829959392547607, | |
| "rewards/margins": 0.025901764631271362, | |
| "rewards/rejected": -0.15420134365558624, | |
| "sft_loss": 1.2829958200454712, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.5981006263891695, | |
| "grad_norm": 1.415165901184082, | |
| "learning_rate": 4.524731551896978e-06, | |
| "logits/chosen": 247.46142578125, | |
| "logits/rejected": 247.42459106445312, | |
| "logps/chosen": -1.2169711589813232, | |
| "logps/rejected": -1.3963136672973633, | |
| "loss": 1.2853, | |
| "odds_ratio_loss": 0.6832239031791687, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12169712781906128, | |
| "rewards/margins": 0.017934244126081467, | |
| "rewards/rejected": -0.13963137567043304, | |
| "sft_loss": 1.2169711589813232, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.6142655081834714, | |
| "grad_norm": 2.7373573780059814, | |
| "learning_rate": 4.4995918590781925e-06, | |
| "logits/chosen": 250.4862518310547, | |
| "logits/rejected": 250.1310272216797, | |
| "logps/chosen": -1.2185784578323364, | |
| "logps/rejected": -1.435258150100708, | |
| "loss": 1.2834, | |
| "odds_ratio_loss": 0.6484395265579224, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12185785919427872, | |
| "rewards/margins": 0.021667957305908203, | |
| "rewards/rejected": -0.14352580904960632, | |
| "sft_loss": 1.2185784578323364, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6304303899777733, | |
| "grad_norm": 1.0431718826293945, | |
| "learning_rate": 4.473877972981797e-06, | |
| "logits/chosen": 247.82730102539062, | |
| "logits/rejected": 248.197998046875, | |
| "logps/chosen": -1.3133275508880615, | |
| "logps/rejected": -1.566138505935669, | |
| "loss": 1.378, | |
| "odds_ratio_loss": 0.6465703248977661, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.13133276998996735, | |
| "rewards/margins": 0.02528109773993492, | |
| "rewards/rejected": -0.15661385655403137, | |
| "sft_loss": 1.3133275508880615, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6465952717720752, | |
| "grad_norm": 2.605905771255493, | |
| "learning_rate": 4.447597277484894e-06, | |
| "logits/chosen": 248.4889678955078, | |
| "logits/rejected": 248.0493927001953, | |
| "logps/chosen": -1.1982879638671875, | |
| "logps/rejected": -1.3909344673156738, | |
| "loss": 1.2662, | |
| "odds_ratio_loss": 0.6788827180862427, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.11982879787683487, | |
| "rewards/margins": 0.01926465705037117, | |
| "rewards/rejected": -0.13909344375133514, | |
| "sft_loss": 1.1982879638671875, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6627601535663771, | |
| "grad_norm": 2.7441976070404053, | |
| "learning_rate": 4.42075731922687e-06, | |
| "logits/chosen": 250.9984893798828, | |
| "logits/rejected": 250.879150390625, | |
| "logps/chosen": -1.3381072282791138, | |
| "logps/rejected": -1.476546049118042, | |
| "loss": 1.4062, | |
| "odds_ratio_loss": 0.6813501119613647, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.13381072878837585, | |
| "rewards/margins": 0.013843873515725136, | |
| "rewards/rejected": -0.14765460789203644, | |
| "sft_loss": 1.3381072282791138, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.6789250353606789, | |
| "grad_norm": 3.2034897804260254, | |
| "learning_rate": 4.3933658054423465e-06, | |
| "logits/chosen": 249.34951782226562, | |
| "logits/rejected": 249.37582397460938, | |
| "logps/chosen": -1.2343724966049194, | |
| "logps/rejected": -1.4455711841583252, | |
| "loss": 1.2964, | |
| "odds_ratio_loss": 0.6205655932426453, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.12343724817037582, | |
| "rewards/margins": 0.021119873970746994, | |
| "rewards/rejected": -0.1445571333169937, | |
| "sft_loss": 1.2343724966049194, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.6950899171549808, | |
| "grad_norm": 2.552898645401001, | |
| "learning_rate": 4.365430601748003e-06, | |
| "logits/chosen": 247.7689208984375, | |
| "logits/rejected": 247.95205688476562, | |
| "logps/chosen": -1.3558423519134521, | |
| "logps/rejected": -1.4942983388900757, | |
| "loss": 1.4265, | |
| "odds_ratio_loss": 0.7065616250038147, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.13558423519134521, | |
| "rewards/margins": 0.013845594599843025, | |
| "rewards/rejected": -0.1494298279285431, | |
| "sft_loss": 1.3558423519134521, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.7112547989492827, | |
| "grad_norm": 7.701834201812744, | |
| "learning_rate": 4.336959729883925e-06, | |
| "logits/chosen": 248.16812133789062, | |
| "logits/rejected": 248.47384643554688, | |
| "logps/chosen": -1.2508445978164673, | |
| "logps/rejected": -1.3401494026184082, | |
| "loss": 1.3242, | |
| "odds_ratio_loss": 0.7333300113677979, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12508445978164673, | |
| "rewards/margins": 0.008930487558245659, | |
| "rewards/rejected": -0.13401496410369873, | |
| "sft_loss": 1.2508445978164673, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.7274196807435845, | |
| "grad_norm": 1.3677743673324585, | |
| "learning_rate": 4.307961365410118e-06, | |
| "logits/chosen": 249.19546508789062, | |
| "logits/rejected": 249.5364990234375, | |
| "logps/chosen": -1.2851893901824951, | |
| "logps/rejected": -1.4277610778808594, | |
| "loss": 1.3525, | |
| "odds_ratio_loss": 0.6732175946235657, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12851892411708832, | |
| "rewards/margins": 0.014257180504500866, | |
| "rewards/rejected": -0.14277611672878265, | |
| "sft_loss": 1.2851893901824951, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7435845625378864, | |
| "grad_norm": 3.3310444355010986, | |
| "learning_rate": 4.278443835358854e-06, | |
| "logits/chosen": 249.6570281982422, | |
| "logits/rejected": 249.6079864501953, | |
| "logps/chosen": -1.1893627643585205, | |
| "logps/rejected": -1.4945032596588135, | |
| "loss": 1.2482, | |
| "odds_ratio_loss": 0.5885173082351685, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.11893627792596817, | |
| "rewards/margins": 0.030514035373926163, | |
| "rewards/rejected": -0.14945031702518463, | |
| "sft_loss": 1.1893627643585205, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7597494443321883, | |
| "grad_norm": 2.5770180225372314, | |
| "learning_rate": 4.248415615843523e-06, | |
| "logits/chosen": 249.5537567138672, | |
| "logits/rejected": 249.5972442626953, | |
| "logps/chosen": -1.2710212469100952, | |
| "logps/rejected": -1.4037456512451172, | |
| "loss": 1.3415, | |
| "odds_ratio_loss": 0.7046067714691162, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12710212171077728, | |
| "rewards/margins": 0.01327243447303772, | |
| "rewards/rejected": -0.140374556183815, | |
| "sft_loss": 1.2710212469100952, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.7759143261264903, | |
| "grad_norm": 9.182385444641113, | |
| "learning_rate": 4.217885329624666e-06, | |
| "logits/chosen": 249.1255645751953, | |
| "logits/rejected": 249.16854858398438, | |
| "logps/chosen": -1.1571811437606812, | |
| "logps/rejected": -1.4825657606124878, | |
| "loss": 1.2175, | |
| "odds_ratio_loss": 0.6027355194091797, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.11571812629699707, | |
| "rewards/margins": 0.032538462430238724, | |
| "rewards/rejected": -0.1482565701007843, | |
| "sft_loss": 1.1571811437606812, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.7920792079207921, | |
| "grad_norm": 2.0430970191955566, | |
| "learning_rate": 4.186861743633911e-06, | |
| "logits/chosen": 248.51168823242188, | |
| "logits/rejected": 248.83370971679688, | |
| "logps/chosen": -1.216133713722229, | |
| "logps/rejected": -1.4709254503250122, | |
| "loss": 1.2856, | |
| "odds_ratio_loss": 0.694364070892334, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12161336094141006, | |
| "rewards/margins": 0.025479182600975037, | |
| "rewards/rejected": -0.1470925360918045, | |
| "sft_loss": 1.216133713722229, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "grad_norm": 2.13413143157959, | |
| "learning_rate": 4.155353766456497e-06, | |
| "logits/chosen": 252.05142211914062, | |
| "logits/rejected": 251.9636993408203, | |
| "logps/chosen": -1.3067327737808228, | |
| "logps/rejected": -1.4753313064575195, | |
| "loss": 1.374, | |
| "odds_ratio_loss": 0.6729229688644409, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.13067328929901123, | |
| "rewards/margins": 0.01685984991490841, | |
| "rewards/rejected": -0.147533118724823, | |
| "sft_loss": 1.3067327737808228, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "eval_logits/chosen": 249.61227416992188, | |
| "eval_logits/rejected": 249.90635681152344, | |
| "eval_logps/chosen": -1.2762008905410767, | |
| "eval_logps/rejected": -1.5033098459243774, | |
| "eval_loss": 1.3435848951339722, | |
| "eval_odds_ratio_loss": 0.6738389730453491, | |
| "eval_rewards/accuracies": 0.5672727227210999, | |
| "eval_rewards/chosen": -0.12762011587619781, | |
| "eval_rewards/margins": 0.0227108895778656, | |
| "eval_rewards/rejected": -0.15033100545406342, | |
| "eval_runtime": 221.4313, | |
| "eval_samples_per_second": 4.968, | |
| "eval_sft_loss": 1.2762008905410767, | |
| "eval_steps_per_second": 2.484, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8244089715093958, | |
| "grad_norm": 2.4113426208496094, | |
| "learning_rate": 4.123370445773134e-06, | |
| "logits/chosen": 250.3010711669922, | |
| "logits/rejected": 250.5003662109375, | |
| "logps/chosen": -1.2399041652679443, | |
| "logps/rejected": -1.3490018844604492, | |
| "loss": 1.3103, | |
| "odds_ratio_loss": 0.7042885422706604, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12399041652679443, | |
| "rewards/margins": 0.010909780859947205, | |
| "rewards/rejected": -0.13490018248558044, | |
| "sft_loss": 1.2399041652679443, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8405738533036977, | |
| "grad_norm": 4.632988452911377, | |
| "learning_rate": 4.090920965761906e-06, | |
| "logits/chosen": 249.44210815429688, | |
| "logits/rejected": 249.96994018554688, | |
| "logps/chosen": -1.2807283401489258, | |
| "logps/rejected": -1.4942976236343384, | |
| "loss": 1.3484, | |
| "odds_ratio_loss": 0.6771414875984192, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.1280728280544281, | |
| "rewards/margins": 0.02135692723095417, | |
| "rewards/rejected": -0.14942976832389832, | |
| "sft_loss": 1.2807283401489258, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8567387350979996, | |
| "grad_norm": 9.196592330932617, | |
| "learning_rate": 4.058014644460991e-06, | |
| "logits/chosen": 250.1853790283203, | |
| "logits/rejected": 250.6529083251953, | |
| "logps/chosen": -1.2633569240570068, | |
| "logps/rejected": -1.4294393062591553, | |
| "loss": 1.329, | |
| "odds_ratio_loss": 0.6560603976249695, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.12633569538593292, | |
| "rewards/margins": 0.01660825125873089, | |
| "rewards/rejected": -0.14294394850730896, | |
| "sft_loss": 1.2633569240570068, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.8729036168923014, | |
| "grad_norm": 1.8403383493423462, | |
| "learning_rate": 4.024660931092939e-06, | |
| "logits/chosen": 250.708251953125, | |
| "logits/rejected": 251.0161895751953, | |
| "logps/chosen": -1.287913203239441, | |
| "logps/rejected": -1.553476095199585, | |
| "loss": 1.3531, | |
| "odds_ratio_loss": 0.6521779298782349, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12879131734371185, | |
| "rewards/margins": 0.026556288823485374, | |
| "rewards/rejected": -0.15534761548042297, | |
| "sft_loss": 1.287913203239441, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.8890684986866033, | |
| "grad_norm": 7.186382293701172, | |
| "learning_rate": 3.990869403351272e-06, | |
| "logits/chosen": 251.8153839111328, | |
| "logits/rejected": 251.8900604248047, | |
| "logps/chosen": -1.268169641494751, | |
| "logps/rejected": -1.511528491973877, | |
| "loss": 1.3283, | |
| "odds_ratio_loss": 0.6014903783798218, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.12681695818901062, | |
| "rewards/margins": 0.024335889145731926, | |
| "rewards/rejected": -0.1511528491973877, | |
| "sft_loss": 1.268169641494751, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9052333804809052, | |
| "grad_norm": 2.923800230026245, | |
| "learning_rate": 3.956649764650206e-06, | |
| "logits/chosen": 250.7381591796875, | |
| "logits/rejected": 250.7707061767578, | |
| "logps/chosen": -1.2698795795440674, | |
| "logps/rejected": -1.4995825290679932, | |
| "loss": 1.3379, | |
| "odds_ratio_loss": 0.6804186105728149, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12698796391487122, | |
| "rewards/margins": 0.0229702927172184, | |
| "rewards/rejected": -0.14995825290679932, | |
| "sft_loss": 1.2698795795440674, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.9213982622752072, | |
| "grad_norm": 6.1557416915893555, | |
| "learning_rate": 3.92201184133826e-06, | |
| "logits/chosen": 250.94808959960938, | |
| "logits/rejected": 251.642822265625, | |
| "logps/chosen": -1.2907052040100098, | |
| "logps/rejected": -1.54143226146698, | |
| "loss": 1.3538, | |
| "odds_ratio_loss": 0.6311336755752563, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.12907053530216217, | |
| "rewards/margins": 0.025072699412703514, | |
| "rewards/rejected": -0.15414324402809143, | |
| "sft_loss": 1.2907052040100098, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.937563144069509, | |
| "grad_norm": 2.1665000915527344, | |
| "learning_rate": 3.886965579876572e-06, | |
| "logits/chosen": 252.3577423095703, | |
| "logits/rejected": 252.0865478515625, | |
| "logps/chosen": -1.2575817108154297, | |
| "logps/rejected": -1.3686320781707764, | |
| "loss": 1.3293, | |
| "odds_ratio_loss": 0.7170482873916626, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12575815618038177, | |
| "rewards/margins": 0.011105048470199108, | |
| "rewards/rejected": -0.13686320185661316, | |
| "sft_loss": 1.2575817108154297, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9537280258638109, | |
| "grad_norm": 2.289733648300171, | |
| "learning_rate": 3.851521043982716e-06, | |
| "logits/chosen": 251.7819061279297, | |
| "logits/rejected": 251.52749633789062, | |
| "logps/chosen": -1.2542387247085571, | |
| "logps/rejected": -1.3954790830612183, | |
| "loss": 1.3206, | |
| "odds_ratio_loss": 0.6639243960380554, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.1254238784313202, | |
| "rewards/margins": 0.014124047942459583, | |
| "rewards/rejected": -0.1395479142665863, | |
| "sft_loss": 1.2542387247085571, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.9698929076581128, | |
| "grad_norm": 2.7564313411712646, | |
| "learning_rate": 3.81568841174086e-06, | |
| "logits/chosen": 251.03280639648438, | |
| "logits/rejected": 251.2174835205078, | |
| "logps/chosen": -1.2807530164718628, | |
| "logps/rejected": -1.5129293203353882, | |
| "loss": 1.3482, | |
| "odds_ratio_loss": 0.674277663230896, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12807528674602509, | |
| "rewards/margins": 0.023217635229229927, | |
| "rewards/rejected": -0.15129292011260986, | |
| "sft_loss": 1.2807530164718628, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.9860577894524146, | |
| "grad_norm": 2.1846888065338135, | |
| "learning_rate": 3.7794779726790664e-06, | |
| "logits/chosen": 249.8391571044922, | |
| "logits/rejected": 250.3789520263672, | |
| "logps/chosen": -1.1555012464523315, | |
| "logps/rejected": -1.3768011331558228, | |
| "loss": 1.2212, | |
| "odds_ratio_loss": 0.6573610305786133, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11555011570453644, | |
| "rewards/margins": 0.022129978984594345, | |
| "rewards/rejected": -0.13768009841442108, | |
| "sft_loss": 1.1555012464523315, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.0022226712467166, | |
| "grad_norm": 2.2191011905670166, | |
| "learning_rate": 3.7429001248146096e-06, | |
| "logits/chosen": 250.8198699951172, | |
| "logits/rejected": 251.24819946289062, | |
| "logps/chosen": -1.272541880607605, | |
| "logps/rejected": -1.5292177200317383, | |
| "loss": 1.3338, | |
| "odds_ratio_loss": 0.6125348806381226, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.1272541880607605, | |
| "rewards/margins": 0.0256675872951746, | |
| "rewards/rejected": -0.15292176604270935, | |
| "sft_loss": 1.272541880607605, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.0183875530410185, | |
| "grad_norm": 1.6834843158721924, | |
| "learning_rate": 3.7059653716681227e-06, | |
| "logits/chosen": 250.3338623046875, | |
| "logits/rejected": 250.6593780517578, | |
| "logps/chosen": -1.2664134502410889, | |
| "logps/rejected": -1.469812035560608, | |
| "loss": 1.3343, | |
| "odds_ratio_loss": 0.6792756915092468, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12664134800434113, | |
| "rewards/margins": 0.020339861512184143, | |
| "rewards/rejected": -0.14698120951652527, | |
| "sft_loss": 1.2664134502410889, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.0345524348353203, | |
| "grad_norm": 5.188844203948975, | |
| "learning_rate": 3.668684319247463e-06, | |
| "logits/chosen": 249.46969604492188, | |
| "logits/rejected": 250.1366729736328, | |
| "logps/chosen": -1.1969501972198486, | |
| "logps/rejected": -1.5598738193511963, | |
| "loss": 1.2558, | |
| "odds_ratio_loss": 0.588589072227478, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.11969500780105591, | |
| "rewards/margins": 0.03629238158464432, | |
| "rewards/rejected": -0.15598741173744202, | |
| "sft_loss": 1.1969501972198486, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.0507173166296222, | |
| "grad_norm": 1.8501890897750854, | |
| "learning_rate": 3.6310676730021373e-06, | |
| "logits/chosen": 250.78857421875, | |
| "logits/rejected": 250.8007354736328, | |
| "logps/chosen": -1.2203996181488037, | |
| "logps/rejected": -1.3524749279022217, | |
| "loss": 1.2867, | |
| "odds_ratio_loss": 0.662962794303894, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12203995883464813, | |
| "rewards/margins": 0.013207539916038513, | |
| "rewards/rejected": -0.13524749875068665, | |
| "sft_loss": 1.2203996181488037, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.066882198423924, | |
| "grad_norm": 3.5492091178894043, | |
| "learning_rate": 3.593126234749178e-06, | |
| "logits/chosen": 250.8761749267578, | |
| "logits/rejected": 251.28622436523438, | |
| "logps/chosen": -1.2661250829696655, | |
| "logps/rejected": -1.455129861831665, | |
| "loss": 1.3334, | |
| "odds_ratio_loss": 0.6727336645126343, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12661252915859222, | |
| "rewards/margins": 0.01890045776963234, | |
| "rewards/rejected": -0.14551296830177307, | |
| "sft_loss": 1.2661250829696655, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.083047080218226, | |
| "grad_norm": 3.5715062618255615, | |
| "learning_rate": 3.554870899571343e-06, | |
| "logits/chosen": 252.4844512939453, | |
| "logits/rejected": 252.82699584960938, | |
| "logps/chosen": -1.2469182014465332, | |
| "logps/rejected": -1.4401594400405884, | |
| "loss": 1.3131, | |
| "odds_ratio_loss": 0.6617658734321594, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12469182908535004, | |
| "rewards/margins": 0.019324112683534622, | |
| "rewards/rejected": -0.14401593804359436, | |
| "sft_loss": 1.2469182014465332, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.0992119620125278, | |
| "grad_norm": 4.318095684051514, | |
| "learning_rate": 3.5163126526885373e-06, | |
| "logits/chosen": 252.0143585205078, | |
| "logits/rejected": 251.80081176757812, | |
| "logps/chosen": -1.1914775371551514, | |
| "logps/rejected": -1.4009137153625488, | |
| "loss": 1.2573, | |
| "odds_ratio_loss": 0.6579803824424744, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.11914775520563126, | |
| "rewards/margins": 0.02094360813498497, | |
| "rewards/rejected": -0.14009135961532593, | |
| "sft_loss": 1.1914775371551514, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.1153768438068297, | |
| "grad_norm": 2.403775930404663, | |
| "learning_rate": 3.4774625663033484e-06, | |
| "logits/chosen": 251.2095184326172, | |
| "logits/rejected": 251.48507690429688, | |
| "logps/chosen": -1.2048381567001343, | |
| "logps/rejected": -1.3877532482147217, | |
| "loss": 1.27, | |
| "odds_ratio_loss": 0.6517833471298218, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12048381567001343, | |
| "rewards/margins": 0.018291514366865158, | |
| "rewards/rejected": -0.13877533376216888, | |
| "sft_loss": 1.2048381567001343, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.1315417256011315, | |
| "grad_norm": 1.7898093461990356, | |
| "learning_rate": 3.4383317964216067e-06, | |
| "logits/chosen": 252.33316040039062, | |
| "logits/rejected": 252.1842498779297, | |
| "logps/chosen": -1.1471569538116455, | |
| "logps/rejected": -1.306755781173706, | |
| "loss": 1.2157, | |
| "odds_ratio_loss": 0.6855098009109497, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.11471569538116455, | |
| "rewards/margins": 0.01595989242196083, | |
| "rewards/rejected": -0.13067558407783508, | |
| "sft_loss": 1.1471569538116455, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.1477066073954334, | |
| "grad_norm": 3.209373712539673, | |
| "learning_rate": 3.398931579648877e-06, | |
| "logits/chosen": 251.15170288085938, | |
| "logits/rejected": 251.59976196289062, | |
| "logps/chosen": -1.239712119102478, | |
| "logps/rejected": -1.5323327779769897, | |
| "loss": 1.3045, | |
| "odds_ratio_loss": 0.6475890874862671, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12397120893001556, | |
| "rewards/margins": 0.029262065887451172, | |
| "rewards/rejected": -0.15323328971862793, | |
| "sft_loss": 1.239712119102478, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.1638714891897353, | |
| "grad_norm": 2.6601579189300537, | |
| "learning_rate": 3.359273229963813e-06, | |
| "logits/chosen": 250.285400390625, | |
| "logits/rejected": 250.47323608398438, | |
| "logps/chosen": -1.2064179182052612, | |
| "logps/rejected": -1.3739216327667236, | |
| "loss": 1.2742, | |
| "odds_ratio_loss": 0.6774007081985474, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12064179033041, | |
| "rewards/margins": 0.01675037480890751, | |
| "rewards/rejected": -0.13739216327667236, | |
| "sft_loss": 1.2064179182052612, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.1800363709840371, | |
| "grad_norm": 1.836297631263733, | |
| "learning_rate": 3.319368135469285e-06, | |
| "logits/chosen": 251.77334594726562, | |
| "logits/rejected": 252.28701782226562, | |
| "logps/chosen": -1.2479230165481567, | |
| "logps/rejected": -1.4433178901672363, | |
| "loss": 1.3175, | |
| "odds_ratio_loss": 0.6954701542854309, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12479230016469955, | |
| "rewards/margins": 0.019539497792720795, | |
| "rewards/rejected": -0.14433178305625916, | |
| "sft_loss": 1.2479230165481567, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.196201252778339, | |
| "grad_norm": 3.1846110820770264, | |
| "learning_rate": 3.279227755122228e-06, | |
| "logits/chosen": 252.08438110351562, | |
| "logits/rejected": 252.65792846679688, | |
| "logps/chosen": -1.196380376815796, | |
| "logps/rejected": -1.5170973539352417, | |
| "loss": 1.2585, | |
| "odds_ratio_loss": 0.6215213537216187, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.11963804066181183, | |
| "rewards/margins": 0.032071683555841446, | |
| "rewards/rejected": -0.15170973539352417, | |
| "sft_loss": 1.196380376815796, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.2123661345726409, | |
| "grad_norm": 3.024951934814453, | |
| "learning_rate": 3.2388636154431417e-06, | |
| "logits/chosen": 253.1087646484375, | |
| "logits/rejected": 253.23635864257812, | |
| "logps/chosen": -1.3020392656326294, | |
| "logps/rejected": -1.5343925952911377, | |
| "loss": 1.3675, | |
| "odds_ratio_loss": 0.654754638671875, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.13020391762256622, | |
| "rewards/margins": 0.02323536016047001, | |
| "rewards/rejected": -0.1534392535686493, | |
| "sft_loss": 1.3020392656326294, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.2285310163669427, | |
| "grad_norm": 2.166121482849121, | |
| "learning_rate": 3.198287307206192e-06, | |
| "logits/chosen": 251.711669921875, | |
| "logits/rejected": 251.5684356689453, | |
| "logps/chosen": -1.1889938116073608, | |
| "logps/rejected": -1.4522913694381714, | |
| "loss": 1.2499, | |
| "odds_ratio_loss": 0.60938560962677, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.1188993826508522, | |
| "rewards/margins": 0.02632974646985531, | |
| "rewards/rejected": -0.14522913098335266, | |
| "sft_loss": 1.1889938116073608, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.2446958981612446, | |
| "grad_norm": 1.8584887981414795, | |
| "learning_rate": 3.157510482110856e-06, | |
| "logits/chosen": 252.8727569580078, | |
| "logits/rejected": 253.4295654296875, | |
| "logps/chosen": -1.2046940326690674, | |
| "logps/rejected": -1.360910177230835, | |
| "loss": 1.2735, | |
| "odds_ratio_loss": 0.6879505515098572, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12046940624713898, | |
| "rewards/margins": 0.01562163233757019, | |
| "rewards/rejected": -0.13609102368354797, | |
| "sft_loss": 1.2046940326690674, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.2608607799555465, | |
| "grad_norm": 1.6219208240509033, | |
| "learning_rate": 3.116544849436077e-06, | |
| "logits/chosen": 251.80764770507812, | |
| "logits/rejected": 251.75509643554688, | |
| "logps/chosen": -1.3175479173660278, | |
| "logps/rejected": -1.6150630712509155, | |
| "loss": 1.3813, | |
| "odds_ratio_loss": 0.6378855109214783, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.1317548006772995, | |
| "rewards/margins": 0.029751509428024292, | |
| "rewards/rejected": -0.1615062952041626, | |
| "sft_loss": 1.3175479173660278, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.2770256617498483, | |
| "grad_norm": 2.1420071125030518, | |
| "learning_rate": 3.0754021726778848e-06, | |
| "logits/chosen": 252.167724609375, | |
| "logits/rejected": 251.9316864013672, | |
| "logps/chosen": -1.1495087146759033, | |
| "logps/rejected": -1.426129937171936, | |
| "loss": 1.2132, | |
| "odds_ratio_loss": 0.6372426748275757, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.11495087295770645, | |
| "rewards/margins": 0.02766209840774536, | |
| "rewards/rejected": -0.1426129937171936, | |
| "sft_loss": 1.1495087146759033, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.2931905435441502, | |
| "grad_norm": 1.3823323249816895, | |
| "learning_rate": 3.0340942661714463e-06, | |
| "logits/chosen": 252.6959686279297, | |
| "logits/rejected": 252.73464965820312, | |
| "logps/chosen": -1.2912076711654663, | |
| "logps/rejected": -1.4657213687896729, | |
| "loss": 1.3573, | |
| "odds_ratio_loss": 0.6610310673713684, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12912078201770782, | |
| "rewards/margins": 0.017451368272304535, | |
| "rewards/rejected": -0.14657214283943176, | |
| "sft_loss": 1.2912076711654663, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.3093554253384523, | |
| "grad_norm": 3.4516756534576416, | |
| "learning_rate": 2.992632991698512e-06, | |
| "logits/chosen": 250.41928100585938, | |
| "logits/rejected": 250.66513061523438, | |
| "logps/chosen": -1.219699501991272, | |
| "logps/rejected": -1.483235239982605, | |
| "loss": 1.2828, | |
| "odds_ratio_loss": 0.6311507821083069, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.12196997553110123, | |
| "rewards/margins": 0.02635357342660427, | |
| "rewards/rejected": -0.14832353591918945, | |
| "sft_loss": 1.219699501991272, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.3255203071327541, | |
| "grad_norm": 2.465632677078247, | |
| "learning_rate": 2.9510302550812537e-06, | |
| "logits/chosen": 251.94296264648438, | |
| "logits/rejected": 252.61587524414062, | |
| "logps/chosen": -1.144325852394104, | |
| "logps/rejected": -1.4354488849639893, | |
| "loss": 1.2042, | |
| "odds_ratio_loss": 0.5983381271362305, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11443258821964264, | |
| "rewards/margins": 0.02911229059100151, | |
| "rewards/rejected": -0.14354488253593445, | |
| "sft_loss": 1.144325852394104, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.341685188927056, | |
| "grad_norm": 3.969513416290283, | |
| "learning_rate": 2.9092980027634325e-06, | |
| "logits/chosen": 251.37832641601562, | |
| "logits/rejected": 251.625244140625, | |
| "logps/chosen": -1.1136391162872314, | |
| "logps/rejected": -1.3801125288009644, | |
| "loss": 1.1766, | |
| "odds_ratio_loss": 0.6293498277664185, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.11136390268802643, | |
| "rewards/margins": 0.026647353544831276, | |
| "rewards/rejected": -0.13801124691963196, | |
| "sft_loss": 1.1136391162872314, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.3578500707213579, | |
| "grad_norm": 1.7552839517593384, | |
| "learning_rate": 2.867448218379927e-06, | |
| "logits/chosen": 252.9868621826172, | |
| "logits/rejected": 253.2499542236328, | |
| "logps/chosen": -1.249079704284668, | |
| "logps/rejected": -1.4685295820236206, | |
| "loss": 1.3139, | |
| "odds_ratio_loss": 0.6482545733451843, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.1249079704284668, | |
| "rewards/margins": 0.02194499969482422, | |
| "rewards/rejected": -0.14685297012329102, | |
| "sft_loss": 1.249079704284668, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.3740149525156597, | |
| "grad_norm": 5.6061906814575195, | |
| "learning_rate": 2.825492919315559e-06, | |
| "logits/chosen": 252.72372436523438, | |
| "logits/rejected": 252.32168579101562, | |
| "logps/chosen": -1.2922828197479248, | |
| "logps/rejected": -1.4327681064605713, | |
| "loss": 1.3613, | |
| "odds_ratio_loss": 0.6897528767585754, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12922829389572144, | |
| "rewards/margins": 0.01404851209372282, | |
| "rewards/rejected": -0.14327679574489594, | |
| "sft_loss": 1.2922828197479248, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.3901798343099616, | |
| "grad_norm": 2.2057290077209473, | |
| "learning_rate": 2.7834441532542482e-06, | |
| "logits/chosen": 251.51272583007812, | |
| "logits/rejected": 251.97573852539062, | |
| "logps/chosen": -1.1630654335021973, | |
| "logps/rejected": -1.4224598407745361, | |
| "loss": 1.2262, | |
| "odds_ratio_loss": 0.6317997574806213, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.11630652844905853, | |
| "rewards/margins": 0.025939440354704857, | |
| "rewards/rejected": -0.14224597811698914, | |
| "sft_loss": 1.1630654335021973, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.4063447161042635, | |
| "grad_norm": 2.0599286556243896, | |
| "learning_rate": 2.74131399471945e-06, | |
| "logits/chosen": 252.7571258544922, | |
| "logits/rejected": 253.06008911132812, | |
| "logps/chosen": -1.2314178943634033, | |
| "logps/rejected": -1.404909372329712, | |
| "loss": 1.297, | |
| "odds_ratio_loss": 0.6555390357971191, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.12314176559448242, | |
| "rewards/margins": 0.01734915003180504, | |
| "rewards/rejected": -0.14049093425273895, | |
| "sft_loss": 1.2314178943634033, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.4225095978985653, | |
| "grad_norm": 3.7026567459106445, | |
| "learning_rate": 2.6991145416068947e-06, | |
| "logits/chosen": 252.689697265625, | |
| "logits/rejected": 252.87332153320312, | |
| "logps/chosen": -1.2634754180908203, | |
| "logps/rejected": -1.376312255859375, | |
| "loss": 1.3339, | |
| "odds_ratio_loss": 0.7037913799285889, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12634754180908203, | |
| "rewards/margins": 0.011283671483397484, | |
| "rewards/rejected": -0.13763120770454407, | |
| "sft_loss": 1.2634754180908203, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.4386744796928672, | |
| "grad_norm": 2.7741122245788574, | |
| "learning_rate": 2.6568579117106143e-06, | |
| "logits/chosen": 251.893310546875, | |
| "logits/rejected": 251.9792022705078, | |
| "logps/chosen": -1.1909462213516235, | |
| "logps/rejected": -1.444592833518982, | |
| "loss": 1.257, | |
| "odds_ratio_loss": 0.6606670618057251, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.11909462511539459, | |
| "rewards/margins": 0.025364672765135765, | |
| "rewards/rejected": -0.1444592922925949, | |
| "sft_loss": 1.1909462213516235, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.454839361487169, | |
| "grad_norm": 1.2793887853622437, | |
| "learning_rate": 2.6145562392432544e-06, | |
| "logits/chosen": 253.50723266601562, | |
| "logits/rejected": 253.42764282226562, | |
| "logps/chosen": -1.2168656587600708, | |
| "logps/rejected": -1.336360216140747, | |
| "loss": 1.2887, | |
| "odds_ratio_loss": 0.7181415557861328, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12168655544519424, | |
| "rewards/margins": 0.01194946188479662, | |
| "rewards/rejected": -0.13363602757453918, | |
| "sft_loss": 1.2168656587600708, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.471004243281471, | |
| "grad_norm": 2.857558012008667, | |
| "learning_rate": 2.5722216713516682e-06, | |
| "logits/chosen": 252.78237915039062, | |
| "logits/rejected": 253.788330078125, | |
| "logps/chosen": -1.1416139602661133, | |
| "logps/rejected": -1.3757555484771729, | |
| "loss": 1.2043, | |
| "odds_ratio_loss": 0.6271349787712097, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.11416139453649521, | |
| "rewards/margins": 0.023414146155118942, | |
| "rewards/rejected": -0.13757555186748505, | |
| "sft_loss": 1.1416139602661133, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.4871691250757728, | |
| "grad_norm": 2.625776529312134, | |
| "learning_rate": 2.5298663646288064e-06, | |
| "logits/chosen": 253.61221313476562, | |
| "logits/rejected": 254.0120391845703, | |
| "logps/chosen": -1.1546480655670166, | |
| "logps/rejected": -1.4036109447479248, | |
| "loss": 1.2201, | |
| "odds_ratio_loss": 0.6542297601699829, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11546480655670166, | |
| "rewards/margins": 0.024896297603845596, | |
| "rewards/rejected": -0.14036110043525696, | |
| "sft_loss": 1.1546480655670166, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.503334006870075, | |
| "grad_norm": 3.928030014038086, | |
| "learning_rate": 2.487502481622879e-06, | |
| "logits/chosen": 252.84619140625, | |
| "logits/rejected": 253.71127319335938, | |
| "logps/chosen": -1.2712576389312744, | |
| "logps/rejected": -1.42746901512146, | |
| "loss": 1.3413, | |
| "odds_ratio_loss": 0.7003083229064941, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12712574005126953, | |
| "rewards/margins": 0.01562117226421833, | |
| "rewards/rejected": -0.1427469402551651, | |
| "sft_loss": 1.2712576389312744, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.5194988886643768, | |
| "grad_norm": 2.4900426864624023, | |
| "learning_rate": 2.4451421873448253e-06, | |
| "logits/chosen": 252.51846313476562, | |
| "logits/rejected": 253.07400512695312, | |
| "logps/chosen": -1.193199634552002, | |
| "logps/rejected": -1.3677222728729248, | |
| "loss": 1.2601, | |
| "odds_ratio_loss": 0.6688076257705688, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11931997537612915, | |
| "rewards/margins": 0.01745227724313736, | |
| "rewards/rejected": -0.1367722451686859, | |
| "sft_loss": 1.193199634552002, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.5356637704586786, | |
| "grad_norm": 6.85699987411499, | |
| "learning_rate": 2.40279764577506e-06, | |
| "logits/chosen": 253.85693359375, | |
| "logits/rejected": 253.9010467529297, | |
| "logps/chosen": -1.304840087890625, | |
| "logps/rejected": -1.417873501777649, | |
| "loss": 1.3741, | |
| "odds_ratio_loss": 0.6923686861991882, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.13048401474952698, | |
| "rewards/margins": 0.011303339153528214, | |
| "rewards/rejected": -0.1417873501777649, | |
| "sft_loss": 1.304840087890625, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.5518286522529805, | |
| "grad_norm": 2.3570547103881836, | |
| "learning_rate": 2.3604810163705242e-06, | |
| "logits/chosen": 253.90060424804688, | |
| "logits/rejected": 254.25430297851562, | |
| "logps/chosen": -1.1358963251113892, | |
| "logps/rejected": -1.3512394428253174, | |
| "loss": 1.1966, | |
| "odds_ratio_loss": 0.6068128943443298, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.11358962953090668, | |
| "rewards/margins": 0.021534323692321777, | |
| "rewards/rejected": -0.13512396812438965, | |
| "sft_loss": 1.1358963251113892, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.5679935340472824, | |
| "grad_norm": 1.6715513467788696, | |
| "learning_rate": 2.3182044505730364e-06, | |
| "logits/chosen": 252.765380859375, | |
| "logits/rejected": 252.7443389892578, | |
| "logps/chosen": -1.0937732458114624, | |
| "logps/rejected": -1.302191972732544, | |
| "loss": 1.1567, | |
| "odds_ratio_loss": 0.6288636922836304, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.10937733948230743, | |
| "rewards/margins": 0.020841870456933975, | |
| "rewards/rejected": -0.13021919131278992, | |
| "sft_loss": 1.0937732458114624, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.5841584158415842, | |
| "grad_norm": 1.8489584922790527, | |
| "learning_rate": 2.275980088319941e-06, | |
| "logits/chosen": 253.30712890625, | |
| "logits/rejected": 253.5155487060547, | |
| "logps/chosen": -1.149460792541504, | |
| "logps/rejected": -1.2745110988616943, | |
| "loss": 1.2198, | |
| "odds_ratio_loss": 0.7036079168319702, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.11494608223438263, | |
| "rewards/margins": 0.012505029328167439, | |
| "rewards/rejected": -0.1274511069059372, | |
| "sft_loss": 1.149460792541504, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.600323297635886, | |
| "grad_norm": 2.3143341541290283, | |
| "learning_rate": 2.2338200545580577e-06, | |
| "logits/chosen": 253.9146728515625, | |
| "logits/rejected": 254.3609619140625, | |
| "logps/chosen": -1.1358720064163208, | |
| "logps/rejected": -1.409860372543335, | |
| "loss": 1.203, | |
| "odds_ratio_loss": 0.6715231537818909, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.11358718574047089, | |
| "rewards/margins": 0.02739885076880455, | |
| "rewards/rejected": -0.14098605513572693, | |
| "sft_loss": 1.1358720064163208, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "grad_norm": 2.5078933238983154, | |
| "learning_rate": 2.191736455761947e-06, | |
| "logits/chosen": 252.4419708251953, | |
| "logits/rejected": 252.6824493408203, | |
| "logps/chosen": -1.102782964706421, | |
| "logps/rejected": -1.295693039894104, | |
| "loss": 1.1628, | |
| "odds_ratio_loss": 0.5999386310577393, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.11027830839157104, | |
| "rewards/margins": 0.019290992990136147, | |
| "rewards/rejected": -0.12956929206848145, | |
| "sft_loss": 1.102782964706421, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "eval_logits/chosen": 252.82716369628906, | |
| "eval_logits/rejected": 253.18104553222656, | |
| "eval_logps/chosen": -1.2153432369232178, | |
| "eval_logps/rejected": -1.446128010749817, | |
| "eval_loss": 1.2833058834075928, | |
| "eval_odds_ratio_loss": 0.6796271204948425, | |
| "eval_rewards/accuracies": 0.5618181824684143, | |
| "eval_rewards/chosen": -0.1215343102812767, | |
| "eval_rewards/margins": 0.023078490048646927, | |
| "eval_rewards/rejected": -0.14461281895637512, | |
| "eval_runtime": 221.4361, | |
| "eval_samples_per_second": 4.968, | |
| "eval_sft_loss": 1.2153432369232178, | |
| "eval_steps_per_second": 2.484, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.6326530612244898, | |
| "grad_norm": 1.7511672973632812, | |
| "learning_rate": 2.1497413764574673e-06, | |
| "logits/chosen": 253.8401336669922, | |
| "logits/rejected": 253.7457733154297, | |
| "logps/chosen": -1.2121939659118652, | |
| "logps/rejected": -1.4931201934814453, | |
| "loss": 1.2703, | |
| "odds_ratio_loss": 0.5808267593383789, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.12121939659118652, | |
| "rewards/margins": 0.028092628344893456, | |
| "rewards/rejected": -0.14931201934814453, | |
| "sft_loss": 1.2121939659118652, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.6488179430187917, | |
| "grad_norm": 2.1624321937561035, | |
| "learning_rate": 2.1078468757516395e-06, | |
| "logits/chosen": 252.7372589111328, | |
| "logits/rejected": 253.10342407226562, | |
| "logps/chosen": -1.1226885318756104, | |
| "logps/rejected": -1.302170991897583, | |
| "loss": 1.1845, | |
| "odds_ratio_loss": 0.6178861856460571, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.11226886510848999, | |
| "rewards/margins": 0.017948249354958534, | |
| "rewards/rejected": -0.13021712005138397, | |
| "sft_loss": 1.1226885318756104, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.6649828248130936, | |
| "grad_norm": 2.5826563835144043, | |
| "learning_rate": 2.0660649838698145e-06, | |
| "logits/chosen": 255.34326171875, | |
| "logits/rejected": 255.65859985351562, | |
| "logps/chosen": -1.1558864116668701, | |
| "logps/rejected": -1.3295384645462036, | |
| "loss": 1.2211, | |
| "odds_ratio_loss": 0.6525439023971558, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11558864265680313, | |
| "rewards/margins": 0.01736520044505596, | |
| "rewards/rejected": -0.13295385241508484, | |
| "sft_loss": 1.1558864116668701, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.6811477066073954, | |
| "grad_norm": 1.975549340248108, | |
| "learning_rate": 2.0244076987011284e-06, | |
| "logits/chosen": 255.1981964111328, | |
| "logits/rejected": 255.7158966064453, | |
| "logps/chosen": -1.2127221822738647, | |
| "logps/rejected": -1.4685566425323486, | |
| "loss": 1.2727, | |
| "odds_ratio_loss": 0.6000550389289856, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.1212722510099411, | |
| "rewards/margins": 0.025583425536751747, | |
| "rewards/rejected": -0.1468556672334671, | |
| "sft_loss": 1.2127221822738647, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.6973125884016973, | |
| "grad_norm": 2.224191904067993, | |
| "learning_rate": 1.982886982353251e-06, | |
| "logits/chosen": 252.6818389892578, | |
| "logits/rejected": 252.80859375, | |
| "logps/chosen": -1.193681240081787, | |
| "logps/rejected": -1.44672691822052, | |
| "loss": 1.2608, | |
| "odds_ratio_loss": 0.6715336441993713, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.11936812102794647, | |
| "rewards/margins": 0.025304565206170082, | |
| "rewards/rejected": -0.144672691822052, | |
| "sft_loss": 1.193681240081787, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.7134774701959992, | |
| "grad_norm": 2.571403980255127, | |
| "learning_rate": 1.941514757717392e-06, | |
| "logits/chosen": 253.2911376953125, | |
| "logits/rejected": 254.0371551513672, | |
| "logps/chosen": -1.2021260261535645, | |
| "logps/rejected": -1.443331003189087, | |
| "loss": 1.2653, | |
| "odds_ratio_loss": 0.6321113705635071, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.1202125996351242, | |
| "rewards/margins": 0.02412049099802971, | |
| "rewards/rejected": -0.1443330943584442, | |
| "sft_loss": 1.2021260261535645, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.729642351990301, | |
| "grad_norm": 4.061903476715088, | |
| "learning_rate": 1.9003029050445953e-06, | |
| "logits/chosen": 254.00650024414062, | |
| "logits/rejected": 254.38876342773438, | |
| "logps/chosen": -1.2242114543914795, | |
| "logps/rejected": -1.4163745641708374, | |
| "loss": 1.2891, | |
| "odds_ratio_loss": 0.6486276984214783, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.12242114543914795, | |
| "rewards/margins": 0.019216306507587433, | |
| "rewards/rejected": -0.14163745939731598, | |
| "sft_loss": 1.2242114543914795, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.745807233784603, | |
| "grad_norm": 2.371570110321045, | |
| "learning_rate": 1.8592632585342523e-06, | |
| "logits/chosen": 254.29745483398438, | |
| "logits/rejected": 254.67745971679688, | |
| "logps/chosen": -1.1612073183059692, | |
| "logps/rejected": -1.4247183799743652, | |
| "loss": 1.2246, | |
| "odds_ratio_loss": 0.6341406106948853, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11612071841955185, | |
| "rewards/margins": 0.026351114735007286, | |
| "rewards/rejected": -0.1424718201160431, | |
| "sft_loss": 1.1612073183059692, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.7619721155789048, | |
| "grad_norm": 8.819137573242188, | |
| "learning_rate": 1.8184076029358527e-06, | |
| "logits/chosen": 253.06661987304688, | |
| "logits/rejected": 252.38119506835938, | |
| "logps/chosen": -1.161278486251831, | |
| "logps/rejected": -1.2557886838912964, | |
| "loss": 1.2286, | |
| "odds_ratio_loss": 0.6734786033630371, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.1161278635263443, | |
| "rewards/margins": 0.009451002813875675, | |
| "rewards/rejected": -0.1255788505077362, | |
| "sft_loss": 1.161278486251831, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.7781369973732066, | |
| "grad_norm": 1.7281618118286133, | |
| "learning_rate": 1.7777476701649318e-06, | |
| "logits/chosen": 251.3661651611328, | |
| "logits/rejected": 252.11477661132812, | |
| "logps/chosen": -1.1861474514007568, | |
| "logps/rejected": -1.3936630487442017, | |
| "loss": 1.2518, | |
| "odds_ratio_loss": 0.6561599373817444, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11861474812030792, | |
| "rewards/margins": 0.020751552656292915, | |
| "rewards/rejected": -0.1393662989139557, | |
| "sft_loss": 1.1861474514007568, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.7943018791675085, | |
| "grad_norm": 3.3538103103637695, | |
| "learning_rate": 1.7372951359341925e-06, | |
| "logits/chosen": 253.0167236328125, | |
| "logits/rejected": 253.53396606445312, | |
| "logps/chosen": -1.137481451034546, | |
| "logps/rejected": -1.2894176244735718, | |
| "loss": 1.2055, | |
| "odds_ratio_loss": 0.6805364489555359, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.11374815553426743, | |
| "rewards/margins": 0.01519359927624464, | |
| "rewards/rejected": -0.12894175946712494, | |
| "sft_loss": 1.137481451034546, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.8104667609618104, | |
| "grad_norm": 3.6225833892822266, | |
| "learning_rate": 1.6970616164007547e-06, | |
| "logits/chosen": 252.6470489501953, | |
| "logits/rejected": 252.9353485107422, | |
| "logps/chosen": -1.1084340810775757, | |
| "logps/rejected": -1.3413022756576538, | |
| "loss": 1.1728, | |
| "odds_ratio_loss": 0.6432042717933655, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.11084340512752533, | |
| "rewards/margins": 0.02328682318329811, | |
| "rewards/rejected": -0.13413023948669434, | |
| "sft_loss": 1.1084340810775757, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 1.8266316427561122, | |
| "grad_norm": 4.332692623138428, | |
| "learning_rate": 1.6570586648305276e-06, | |
| "logits/chosen": 253.6255645751953, | |
| "logits/rejected": 253.76217651367188, | |
| "logps/chosen": -1.1925103664398193, | |
| "logps/rejected": -1.4342319965362549, | |
| "loss": 1.2579, | |
| "odds_ratio_loss": 0.6541949510574341, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11925105005502701, | |
| "rewards/margins": 0.024172160774469376, | |
| "rewards/rejected": -0.14342321455478668, | |
| "sft_loss": 1.1925103664398193, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 1.842796524550414, | |
| "grad_norm": 3.238105535507202, | |
| "learning_rate": 1.6172977682806151e-06, | |
| "logits/chosen": 253.7568817138672, | |
| "logits/rejected": 254.8863525390625, | |
| "logps/chosen": -1.2200841903686523, | |
| "logps/rejected": -1.4592787027359009, | |
| "loss": 1.2837, | |
| "odds_ratio_loss": 0.6365170478820801, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.12200842052698135, | |
| "rewards/margins": 0.023919429630041122, | |
| "rewards/rejected": -0.14592786133289337, | |
| "sft_loss": 1.2200841903686523, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 1.858961406344716, | |
| "grad_norm": 2.5219290256500244, | |
| "learning_rate": 1.5777903443007586e-06, | |
| "logits/chosen": 253.631103515625, | |
| "logits/rejected": 253.82504272460938, | |
| "logps/chosen": -1.235215425491333, | |
| "logps/rejected": -1.4535129070281982, | |
| "loss": 1.3023, | |
| "odds_ratio_loss": 0.6708552241325378, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12352155148983002, | |
| "rewards/margins": 0.021829739212989807, | |
| "rewards/rejected": -0.14535130560398102, | |
| "sft_loss": 1.235215425491333, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.8751262881390178, | |
| "grad_norm": 3.190958261489868, | |
| "learning_rate": 1.5385477376547226e-06, | |
| "logits/chosen": 255.1521759033203, | |
| "logits/rejected": 255.2525634765625, | |
| "logps/chosen": -1.229001760482788, | |
| "logps/rejected": -1.4793845415115356, | |
| "loss": 1.2891, | |
| "odds_ratio_loss": 0.601111888885498, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.12290020287036896, | |
| "rewards/margins": 0.02503824792802334, | |
| "rewards/rejected": -0.14793843030929565, | |
| "sft_loss": 1.229001760482788, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 1.89129116993332, | |
| "grad_norm": 2.217510461807251, | |
| "learning_rate": 1.4995812170625845e-06, | |
| "logits/chosen": 253.1751251220703, | |
| "logits/rejected": 253.72238159179688, | |
| "logps/chosen": -1.2252581119537354, | |
| "logps/rejected": -1.5921032428741455, | |
| "loss": 1.2851, | |
| "odds_ratio_loss": 0.598137617111206, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12252581119537354, | |
| "rewards/margins": 0.036684513092041016, | |
| "rewards/rejected": -0.15921030938625336, | |
| "sft_loss": 1.2252581119537354, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 1.9074560517276218, | |
| "grad_norm": 3.0452287197113037, | |
| "learning_rate": 1.4609019719648666e-06, | |
| "logits/chosen": 254.07901000976562, | |
| "logits/rejected": 254.59957885742188, | |
| "logps/chosen": -1.2207356691360474, | |
| "logps/rejected": -1.4706141948699951, | |
| "loss": 1.2826, | |
| "odds_ratio_loss": 0.6183902025222778, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.12207356840372086, | |
| "rewards/margins": 0.024987850338220596, | |
| "rewards/rejected": -0.14706142246723175, | |
| "sft_loss": 1.2207356691360474, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 1.9236209335219236, | |
| "grad_norm": 4.679479122161865, | |
| "learning_rate": 1.42252110930943e-06, | |
| "logits/chosen": 252.7305450439453, | |
| "logits/rejected": 252.6374969482422, | |
| "logps/chosen": -1.064835786819458, | |
| "logps/rejected": -1.2910759449005127, | |
| "loss": 1.1283, | |
| "odds_ratio_loss": 0.6346200704574585, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.1064835786819458, | |
| "rewards/margins": 0.02262401580810547, | |
| "rewards/rejected": -0.12910759449005127, | |
| "sft_loss": 1.064835786819458, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 1.9397858153162255, | |
| "grad_norm": 3.286461353302002, | |
| "learning_rate": 1.3844496503620493e-06, | |
| "logits/chosen": 253.310302734375, | |
| "logits/rejected": 253.27023315429688, | |
| "logps/chosen": -1.2112998962402344, | |
| "logps/rejected": -1.3967139720916748, | |
| "loss": 1.2737, | |
| "odds_ratio_loss": 0.6242542862892151, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12112998962402344, | |
| "rewards/margins": 0.01854141615331173, | |
| "rewards/rejected": -0.1396714150905609, | |
| "sft_loss": 1.2112998962402344, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 1.9559506971105274, | |
| "grad_norm": 2.8077545166015625, | |
| "learning_rate": 1.3466985275416081e-06, | |
| "logits/chosen": 254.2769775390625, | |
| "logits/rejected": 254.47360229492188, | |
| "logps/chosen": -1.2563018798828125, | |
| "logps/rejected": -1.4514508247375488, | |
| "loss": 1.3239, | |
| "odds_ratio_loss": 0.6757391691207886, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.1256301999092102, | |
| "rewards/margins": 0.019514882937073708, | |
| "rewards/rejected": -0.14514507353305817, | |
| "sft_loss": 1.2563018798828125, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 1.9721155789048292, | |
| "grad_norm": 2.275397777557373, | |
| "learning_rate": 1.309278581280791e-06, | |
| "logits/chosen": 253.5750274658203, | |
| "logits/rejected": 253.99935913085938, | |
| "logps/chosen": -1.1356334686279297, | |
| "logps/rejected": -1.4314597845077515, | |
| "loss": 1.1934, | |
| "odds_ratio_loss": 0.5772610902786255, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.11356334388256073, | |
| "rewards/margins": 0.029582645744085312, | |
| "rewards/rejected": -0.14314597845077515, | |
| "sft_loss": 1.1356334686279297, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 1.9882804606991311, | |
| "grad_norm": 1.454276204109192, | |
| "learning_rate": 1.272200556913199e-06, | |
| "logits/chosen": 254.544677734375, | |
| "logits/rejected": 254.67251586914062, | |
| "logps/chosen": -1.1884077787399292, | |
| "logps/rejected": -1.395115613937378, | |
| "loss": 1.2599, | |
| "odds_ratio_loss": 0.7147720456123352, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.1188407689332962, | |
| "rewards/margins": 0.020670795813202858, | |
| "rewards/rejected": -0.1395115852355957, | |
| "sft_loss": 1.1884077787399292, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.004445342493433, | |
| "grad_norm": 3.6475422382354736, | |
| "learning_rate": 1.2354751015877698e-06, | |
| "logits/chosen": 252.74777221679688, | |
| "logits/rejected": 253.66641235351562, | |
| "logps/chosen": -1.1167339086532593, | |
| "logps/rejected": -1.4450454711914062, | |
| "loss": 1.1791, | |
| "odds_ratio_loss": 0.6237870454788208, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11167339980602264, | |
| "rewards/margins": 0.03283114731311798, | |
| "rewards/rejected": -0.14450454711914062, | |
| "sft_loss": 1.1167339086532593, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.020610224287735, | |
| "grad_norm": 3.52698016166687, | |
| "learning_rate": 1.1991127612113945e-06, | |
| "logits/chosen": 254.6741943359375, | |
| "logits/rejected": 254.9825897216797, | |
| "logps/chosen": -1.1792643070220947, | |
| "logps/rejected": -1.4326034784317017, | |
| "loss": 1.2387, | |
| "odds_ratio_loss": 0.5942111611366272, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.11792641878128052, | |
| "rewards/margins": 0.02533392235636711, | |
| "rewards/rejected": -0.14326035976409912, | |
| "sft_loss": 1.1792643070220947, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.036775106082037, | |
| "grad_norm": 3.579160690307617, | |
| "learning_rate": 1.1631239774206035e-06, | |
| "logits/chosen": 253.5153350830078, | |
| "logits/rejected": 253.659912109375, | |
| "logps/chosen": -1.1673438549041748, | |
| "logps/rejected": -1.4458467960357666, | |
| "loss": 1.2314, | |
| "odds_ratio_loss": 0.6410170793533325, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.11673440039157867, | |
| "rewards/margins": 0.027850273996591568, | |
| "rewards/rejected": -0.14458468556404114, | |
| "sft_loss": 1.1673438549041748, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.052939987876339, | |
| "grad_norm": 3.0812463760375977, | |
| "learning_rate": 1.1275190845831978e-06, | |
| "logits/chosen": 254.5985870361328, | |
| "logits/rejected": 254.2525177001953, | |
| "logps/chosen": -1.1342524290084839, | |
| "logps/rejected": -1.3948237895965576, | |
| "loss": 1.1925, | |
| "odds_ratio_loss": 0.5824798345565796, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.11342523247003555, | |
| "rewards/margins": 0.026057133451104164, | |
| "rewards/rejected": -0.13948237895965576, | |
| "sft_loss": 1.1342524290084839, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.0691048696706407, | |
| "grad_norm": 2.4549710750579834, | |
| "learning_rate": 1.0923083068306778e-06, | |
| "logits/chosen": 254.7982635498047, | |
| "logits/rejected": 255.1488494873047, | |
| "logps/chosen": -1.1482160091400146, | |
| "logps/rejected": -1.4850049018859863, | |
| "loss": 1.2055, | |
| "odds_ratio_loss": 0.5724589824676514, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.11482159048318863, | |
| "rewards/margins": 0.033678896725177765, | |
| "rewards/rejected": -0.1485004872083664, | |
| "sft_loss": 1.1482160091400146, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.0852697514649425, | |
| "grad_norm": 1.778605580329895, | |
| "learning_rate": 1.0575017551223348e-06, | |
| "logits/chosen": 253.03524780273438, | |
| "logits/rejected": 253.53366088867188, | |
| "logps/chosen": -1.087461233139038, | |
| "logps/rejected": -1.321656584739685, | |
| "loss": 1.1524, | |
| "odds_ratio_loss": 0.6496065855026245, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.10874611139297485, | |
| "rewards/margins": 0.023419544100761414, | |
| "rewards/rejected": -0.13216565549373627, | |
| "sft_loss": 1.087461233139038, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.1014346332592444, | |
| "grad_norm": 1.522445797920227, | |
| "learning_rate": 1.023109424341833e-06, | |
| "logits/chosen": 254.5054168701172, | |
| "logits/rejected": 255.06100463867188, | |
| "logps/chosen": -1.2142359018325806, | |
| "logps/rejected": -1.448194146156311, | |
| "loss": 1.2781, | |
| "odds_ratio_loss": 0.6386287808418274, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.12142357975244522, | |
| "rewards/margins": 0.023395827040076256, | |
| "rewards/rejected": -0.14481940865516663, | |
| "sft_loss": 1.2142359018325806, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.1175995150535463, | |
| "grad_norm": 2.577580690383911, | |
| "learning_rate": 9.891411904271273e-07, | |
| "logits/chosen": 254.14779663085938, | |
| "logits/rejected": 254.121826171875, | |
| "logps/chosen": -1.100303292274475, | |
| "logps/rejected": -1.3276548385620117, | |
| "loss": 1.1632, | |
| "odds_ratio_loss": 0.628852128982544, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.11003033071756363, | |
| "rewards/margins": 0.02273516170680523, | |
| "rewards/rejected": -0.1327655017375946, | |
| "sft_loss": 1.100303292274475, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.133764396847848, | |
| "grad_norm": 1.5676363706588745, | |
| "learning_rate": 9.556068075345363e-07, | |
| "logits/chosen": 255.0603485107422, | |
| "logits/rejected": 255.1541748046875, | |
| "logps/chosen": -1.1494947671890259, | |
| "logps/rejected": -1.3283547163009644, | |
| "loss": 1.2112, | |
| "odds_ratio_loss": 0.6172733306884766, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.11494947969913483, | |
| "rewards/margins": 0.017885997891426086, | |
| "rewards/rejected": -0.1328354775905609, | |
| "sft_loss": 1.1494947671890259, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.14992927864215, | |
| "grad_norm": 1.964956521987915, | |
| "learning_rate": 9.225159052377838e-07, | |
| "logits/chosen": 254.16183471679688, | |
| "logits/rejected": 254.3532257080078, | |
| "logps/chosen": -1.1823852062225342, | |
| "logps/rejected": -1.4285701513290405, | |
| "loss": 1.2468, | |
| "odds_ratio_loss": 0.6443654894828796, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11823852360248566, | |
| "rewards/margins": 0.024618491530418396, | |
| "rewards/rejected": -0.14285701513290405, | |
| "sft_loss": 1.1823852062225342, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.166094160436452, | |
| "grad_norm": 4.320827484130859, | |
| "learning_rate": 8.898779857628184e-07, | |
| "logits/chosen": 253.94775390625, | |
| "logits/rejected": 253.83328247070312, | |
| "logps/chosen": -1.0813744068145752, | |
| "logps/rejected": -1.289052963256836, | |
| "loss": 1.1442, | |
| "odds_ratio_loss": 0.6285432577133179, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.10813745111227036, | |
| "rewards/margins": 0.020767847076058388, | |
| "rewards/rejected": -0.1289052963256836, | |
| "sft_loss": 1.0813744068145752, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.1822590422307537, | |
| "grad_norm": 1.9166721105575562, | |
| "learning_rate": 8.577024212591975e-07, | |
| "logits/chosen": 255.42715454101562, | |
| "logits/rejected": 255.6570281982422, | |
| "logps/chosen": -1.2112232446670532, | |
| "logps/rejected": -1.4022705554962158, | |
| "loss": 1.2754, | |
| "odds_ratio_loss": 0.6421025991439819, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1211223155260086, | |
| "rewards/margins": 0.019104719161987305, | |
| "rewards/rejected": -0.1402270495891571, | |
| "sft_loss": 1.2112232446670532, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.1984239240250556, | |
| "grad_norm": 2.231593370437622, | |
| "learning_rate": 8.259984511088276e-07, | |
| "logits/chosen": 252.95217895507812, | |
| "logits/rejected": 253.24972534179688, | |
| "logps/chosen": -1.1978521347045898, | |
| "logps/rejected": -1.4085915088653564, | |
| "loss": 1.2643, | |
| "odds_ratio_loss": 0.6643570065498352, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.11978521198034286, | |
| "rewards/margins": 0.021073944866657257, | |
| "rewards/rejected": -0.14085917174816132, | |
| "sft_loss": 1.1978521347045898, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.2145888058193575, | |
| "grad_norm": 1.891438364982605, | |
| "learning_rate": 7.947751792728237e-07, | |
| "logits/chosen": 252.89163208007812, | |
| "logits/rejected": 252.9368133544922, | |
| "logps/chosen": -1.1386100053787231, | |
| "logps/rejected": -1.3931357860565186, | |
| "loss": 1.2001, | |
| "odds_ratio_loss": 0.6149393320083618, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.11386100947856903, | |
| "rewards/margins": 0.02545258030295372, | |
| "rewards/rejected": -0.13931360840797424, | |
| "sft_loss": 1.1386100053787231, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.2307536876136593, | |
| "grad_norm": 11.893668174743652, | |
| "learning_rate": 7.640415716772626e-07, | |
| "logits/chosen": 254.87893676757812, | |
| "logits/rejected": 254.9901123046875, | |
| "logps/chosen": -1.2301312685012817, | |
| "logps/rejected": -1.4715359210968018, | |
| "loss": 1.2969, | |
| "odds_ratio_loss": 0.6676316857337952, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12301311641931534, | |
| "rewards/margins": 0.02414046786725521, | |
| "rewards/rejected": -0.1471536010503769, | |
| "sft_loss": 1.2301312685012817, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.246918569407961, | |
| "grad_norm": 1.648759365081787, | |
| "learning_rate": 7.338064536385722e-07, | |
| "logits/chosen": 253.27536010742188, | |
| "logits/rejected": 253.39450073242188, | |
| "logps/chosen": -1.172890543937683, | |
| "logps/rejected": -1.4573774337768555, | |
| "loss": 1.2306, | |
| "odds_ratio_loss": 0.5775946974754333, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.11728904396295547, | |
| "rewards/margins": 0.02844870649278164, | |
| "rewards/rejected": -0.14573773741722107, | |
| "sft_loss": 1.172890543937683, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.263083451202263, | |
| "grad_norm": 2.644590377807617, | |
| "learning_rate": 7.040785073292883e-07, | |
| "logits/chosen": 254.41006469726562, | |
| "logits/rejected": 254.5663299560547, | |
| "logps/chosen": -1.243436336517334, | |
| "logps/rejected": -1.455594778060913, | |
| "loss": 1.3115, | |
| "odds_ratio_loss": 0.6802859902381897, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12434364855289459, | |
| "rewards/margins": 0.021215861663222313, | |
| "rewards/rejected": -0.14555948972702026, | |
| "sft_loss": 1.243436336517334, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.279248332996565, | |
| "grad_norm": 3.2420878410339355, | |
| "learning_rate": 6.748662692849297e-07, | |
| "logits/chosen": 253.18417358398438, | |
| "logits/rejected": 254.11279296875, | |
| "logps/chosen": -1.1471028327941895, | |
| "logps/rejected": -1.5119264125823975, | |
| "loss": 1.2055, | |
| "odds_ratio_loss": 0.5840214490890503, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.11471028625965118, | |
| "rewards/margins": 0.03648235648870468, | |
| "rewards/rejected": -0.15119265019893646, | |
| "sft_loss": 1.1471028327941895, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.295413214790867, | |
| "grad_norm": 4.394900798797607, | |
| "learning_rate": 6.46178127952686e-07, | |
| "logits/chosen": 254.48062133789062, | |
| "logits/rejected": 254.9115753173828, | |
| "logps/chosen": -1.1684550046920776, | |
| "logps/rejected": -1.38850998878479, | |
| "loss": 1.2286, | |
| "odds_ratio_loss": 0.601581871509552, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.1168455109000206, | |
| "rewards/margins": 0.022005509585142136, | |
| "rewards/rejected": -0.13885101675987244, | |
| "sft_loss": 1.1684550046920776, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.3115780965851687, | |
| "grad_norm": 2.162309169769287, | |
| "learning_rate": 6.180223212826289e-07, | |
| "logits/chosen": 253.58633422851562, | |
| "logits/rejected": 253.8734588623047, | |
| "logps/chosen": -1.1496318578720093, | |
| "logps/rejected": -1.364654779434204, | |
| "loss": 1.213, | |
| "odds_ratio_loss": 0.633824348449707, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.11496319621801376, | |
| "rewards/margins": 0.021502288058400154, | |
| "rewards/rejected": -0.13646547496318817, | |
| "sft_loss": 1.1496318578720093, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.3277429783794705, | |
| "grad_norm": 1.522935152053833, | |
| "learning_rate": 5.904069343621443e-07, | |
| "logits/chosen": 255.19082641601562, | |
| "logits/rejected": 255.11474609375, | |
| "logps/chosen": -1.1330249309539795, | |
| "logps/rejected": -1.386264443397522, | |
| "loss": 1.195, | |
| "odds_ratio_loss": 0.6193984746932983, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.11330248415470123, | |
| "rewards/margins": 0.025323981419205666, | |
| "rewards/rejected": -0.13862647116184235, | |
| "sft_loss": 1.1330249309539795, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.3439078601737724, | |
| "grad_norm": 2.982042074203491, | |
| "learning_rate": 5.633398970942544e-07, | |
| "logits/chosen": 254.9903564453125, | |
| "logits/rejected": 255.1248321533203, | |
| "logps/chosen": -1.1471365690231323, | |
| "logps/rejected": -1.3323371410369873, | |
| "loss": 1.2137, | |
| "odds_ratio_loss": 0.6657688617706299, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.114713653922081, | |
| "rewards/margins": 0.01852005161345005, | |
| "rewards/rejected": -0.1332337111234665, | |
| "sft_loss": 1.1471365690231323, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.3600727419680743, | |
| "grad_norm": 3.2461607456207275, | |
| "learning_rate": 5.368289819205069e-07, | |
| "logits/chosen": 254.27847290039062, | |
| "logits/rejected": 255.0779571533203, | |
| "logps/chosen": -1.11297607421875, | |
| "logps/rejected": -1.3122992515563965, | |
| "loss": 1.1805, | |
| "odds_ratio_loss": 0.6752298474311829, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.1112975925207138, | |
| "rewards/margins": 0.019932324066758156, | |
| "rewards/rejected": -0.1312299221754074, | |
| "sft_loss": 1.11297607421875, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.376237623762376, | |
| "grad_norm": 2.7223591804504395, | |
| "learning_rate": 5.108818015890785e-07, | |
| "logits/chosen": 255.47216796875, | |
| "logits/rejected": 255.6534423828125, | |
| "logps/chosen": -1.2367959022521973, | |
| "logps/rejected": -1.4369171857833862, | |
| "loss": 1.3029, | |
| "odds_ratio_loss": 0.6609222888946533, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12367959320545197, | |
| "rewards/margins": 0.020012129098176956, | |
| "rewards/rejected": -0.14369171857833862, | |
| "sft_loss": 1.2367959022521973, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.392402505556678, | |
| "grad_norm": 2.912327527999878, | |
| "learning_rate": 4.855058069687291e-07, | |
| "logits/chosen": 253.0759735107422, | |
| "logits/rejected": 253.6752471923828, | |
| "logps/chosen": -1.111169695854187, | |
| "logps/rejected": -1.4328919649124146, | |
| "loss": 1.1697, | |
| "odds_ratio_loss": 0.5851289629936218, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.11111698299646378, | |
| "rewards/margins": 0.03217221051454544, | |
| "rewards/rejected": -0.1432892084121704, | |
| "sft_loss": 1.111169695854187, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.40856738735098, | |
| "grad_norm": 2.995020627975464, | |
| "learning_rate": 4.607082849092523e-07, | |
| "logits/chosen": 253.9425811767578, | |
| "logits/rejected": 254.0526580810547, | |
| "logps/chosen": -1.2607060670852661, | |
| "logps/rejected": -1.4026780128479004, | |
| "loss": 1.3291, | |
| "odds_ratio_loss": 0.6835006475448608, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12607058882713318, | |
| "rewards/margins": 0.014197212643921375, | |
| "rewards/rejected": -0.14026781916618347, | |
| "sft_loss": 1.2607060670852661, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "grad_norm": 3.760835886001587, | |
| "learning_rate": 4.3649635614901405e-07, | |
| "logits/chosen": 254.07601928710938, | |
| "logits/rejected": 254.50985717773438, | |
| "logps/chosen": -1.1233417987823486, | |
| "logps/rejected": -1.2859314680099487, | |
| "loss": 1.1874, | |
| "odds_ratio_loss": 0.6403074860572815, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.11233416944742203, | |
| "rewards/margins": 0.01625899039208889, | |
| "rewards/rejected": -0.12859316170215607, | |
| "sft_loss": 1.1233417987823486, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "eval_logits/chosen": 253.6037139892578, | |
| "eval_logits/rejected": 253.95994567871094, | |
| "eval_logps/chosen": -1.1982638835906982, | |
| "eval_logps/rejected": -1.4377323389053345, | |
| "eval_loss": 1.265723466873169, | |
| "eval_odds_ratio_loss": 0.6745957732200623, | |
| "eval_rewards/accuracies": 0.5699999928474426, | |
| "eval_rewards/chosen": -0.11982638388872147, | |
| "eval_rewards/margins": 0.023946860805153847, | |
| "eval_rewards/rejected": -0.14377322793006897, | |
| "eval_runtime": 221.0804, | |
| "eval_samples_per_second": 4.976, | |
| "eval_sft_loss": 1.1982638835906982, | |
| "eval_steps_per_second": 2.488, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4408971509395836, | |
| "grad_norm": 2.074381113052368, | |
| "learning_rate": 4.128769732701973e-07, | |
| "logits/chosen": 254.7397918701172, | |
| "logits/rejected": 254.7943572998047, | |
| "logps/chosen": -1.1791332960128784, | |
| "logps/rejected": -1.4214551448822021, | |
| "loss": 1.2445, | |
| "odds_ratio_loss": 0.6540807485580444, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.11791334301233292, | |
| "rewards/margins": 0.02423218823969364, | |
| "rewards/rejected": -0.14214551448822021, | |
| "sft_loss": 1.1791332960128784, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.4570620327338855, | |
| "grad_norm": 3.3741965293884277, | |
| "learning_rate": 3.8985691870233046e-07, | |
| "logits/chosen": 254.44534301757812, | |
| "logits/rejected": 254.93307495117188, | |
| "logps/chosen": -1.209789514541626, | |
| "logps/rejected": -1.4953523874282837, | |
| "loss": 1.2744, | |
| "odds_ratio_loss": 0.645904004573822, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.1209789514541626, | |
| "rewards/margins": 0.02855629101395607, | |
| "rewards/rejected": -0.14953525364398956, | |
| "sft_loss": 1.209789514541626, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.4732269145281873, | |
| "grad_norm": 4.19984245300293, | |
| "learning_rate": 3.6744280277467904e-07, | |
| "logits/chosen": 253.19186401367188, | |
| "logits/rejected": 253.959228515625, | |
| "logps/chosen": -1.1898527145385742, | |
| "logps/rejected": -1.418869972229004, | |
| "loss": 1.2551, | |
| "odds_ratio_loss": 0.6521813273429871, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11898528039455414, | |
| "rewards/margins": 0.022901728749275208, | |
| "rewards/rejected": -0.14188699424266815, | |
| "sft_loss": 1.1898527145385742, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.489391796322489, | |
| "grad_norm": 3.145732879638672, | |
| "learning_rate": 3.456410618180503e-07, | |
| "logits/chosen": 252.92282104492188, | |
| "logits/rejected": 253.6022491455078, | |
| "logps/chosen": -1.0681793689727783, | |
| "logps/rejected": -1.446299433708191, | |
| "loss": 1.1295, | |
| "odds_ratio_loss": 0.6130428910255432, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.10681793838739395, | |
| "rewards/margins": 0.0378120057284832, | |
| "rewards/rejected": -0.14462995529174805, | |
| "sft_loss": 1.0681793689727783, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.5055566781167915, | |
| "grad_norm": 2.4530389308929443, | |
| "learning_rate": 3.244579563165753e-07, | |
| "logits/chosen": 252.8817138671875, | |
| "logits/rejected": 253.0416259765625, | |
| "logps/chosen": -1.1208980083465576, | |
| "logps/rejected": -1.4661897420883179, | |
| "loss": 1.1805, | |
| "odds_ratio_loss": 0.5958081483840942, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.11208979785442352, | |
| "rewards/margins": 0.034529171884059906, | |
| "rewards/rejected": -0.14661899209022522, | |
| "sft_loss": 1.1208980083465576, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.521721559911093, | |
| "grad_norm": 1.9151691198349, | |
| "learning_rate": 3.038995691099697e-07, | |
| "logits/chosen": 252.8979949951172, | |
| "logits/rejected": 253.34262084960938, | |
| "logps/chosen": -1.2310686111450195, | |
| "logps/rejected": -1.493896484375, | |
| "loss": 1.2954, | |
| "odds_ratio_loss": 0.6432778239250183, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.12310687452554703, | |
| "rewards/margins": 0.02628278359770775, | |
| "rewards/rejected": -0.14938965439796448, | |
| "sft_loss": 1.2310686111450195, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.5378864417053952, | |
| "grad_norm": 4.0270304679870605, | |
| "learning_rate": 2.839718036468192e-07, | |
| "logits/chosen": 255.1189422607422, | |
| "logits/rejected": 255.6085662841797, | |
| "logps/chosen": -1.2376972436904907, | |
| "logps/rejected": -1.4336402416229248, | |
| "loss": 1.306, | |
| "odds_ratio_loss": 0.6828280091285706, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12376973778009415, | |
| "rewards/margins": 0.019594285637140274, | |
| "rewards/rejected": -0.14336401224136353, | |
| "sft_loss": 1.2376972436904907, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.5540513234996967, | |
| "grad_norm": 2.8464980125427246, | |
| "learning_rate": 2.646803822893723e-07, | |
| "logits/chosen": 254.5944366455078, | |
| "logits/rejected": 254.6045379638672, | |
| "logps/chosen": -1.1911519765853882, | |
| "logps/rejected": -1.4335906505584717, | |
| "loss": 1.255, | |
| "odds_ratio_loss": 0.6387141346931458, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.1191151887178421, | |
| "rewards/margins": 0.024243878200650215, | |
| "rewards/rejected": -0.14335909485816956, | |
| "sft_loss": 1.1911519765853882, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.570216205293999, | |
| "grad_norm": 2.3468587398529053, | |
| "learning_rate": 2.460308446703341e-07, | |
| "logits/chosen": 255.0273895263672, | |
| "logits/rejected": 255.33963012695312, | |
| "logps/chosen": -1.1678217649459839, | |
| "logps/rejected": -1.3280802965164185, | |
| "loss": 1.2319, | |
| "odds_ratio_loss": 0.6405949592590332, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.11678217351436615, | |
| "rewards/margins": 0.016025854274630547, | |
| "rewards/rejected": -0.13280804455280304, | |
| "sft_loss": 1.1678217649459839, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.5863810870883004, | |
| "grad_norm": 2.52286434173584, | |
| "learning_rate": 2.2802854610213143e-07, | |
| "logits/chosen": 253.905517578125, | |
| "logits/rejected": 254.2148895263672, | |
| "logps/chosen": -1.0991406440734863, | |
| "logps/rejected": -1.5185635089874268, | |
| "loss": 1.1551, | |
| "odds_ratio_loss": 0.5599113702774048, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.10991404950618744, | |
| "rewards/margins": 0.041942298412323, | |
| "rewards/rejected": -0.15185634791851044, | |
| "sft_loss": 1.0991406440734863, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.6025459688826027, | |
| "grad_norm": 6.791391849517822, | |
| "learning_rate": 2.106786560391072e-07, | |
| "logits/chosen": 253.763671875, | |
| "logits/rejected": 253.8026580810547, | |
| "logps/chosen": -1.2003083229064941, | |
| "logps/rejected": -1.4066941738128662, | |
| "loss": 1.2644, | |
| "odds_ratio_loss": 0.6404808163642883, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.12003083527088165, | |
| "rewards/margins": 0.020638594403862953, | |
| "rewards/rejected": -0.14066943526268005, | |
| "sft_loss": 1.2003083229064941, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.6187108506769046, | |
| "grad_norm": 2.421247720718384, | |
| "learning_rate": 1.9398615659308255e-07, | |
| "logits/chosen": 254.5044403076172, | |
| "logits/rejected": 255.14804077148438, | |
| "logps/chosen": -1.145989179611206, | |
| "logps/rejected": -1.2900917530059814, | |
| "loss": 1.2124, | |
| "odds_ratio_loss": 0.663682222366333, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11459891498088837, | |
| "rewards/margins": 0.014410244300961494, | |
| "rewards/rejected": -0.1290091574192047, | |
| "sft_loss": 1.145989179611206, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.6348757324712064, | |
| "grad_norm": 2.5792062282562256, | |
| "learning_rate": 1.7795584110272184e-07, | |
| "logits/chosen": 254.8577117919922, | |
| "logits/rejected": 254.72903442382812, | |
| "logps/chosen": -1.1733181476593018, | |
| "logps/rejected": -1.3872268199920654, | |
| "loss": 1.2372, | |
| "odds_ratio_loss": 0.6392764449119568, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.11733181774616241, | |
| "rewards/margins": 0.021390849724411964, | |
| "rewards/rejected": -0.13872265815734863, | |
| "sft_loss": 1.1733181476593018, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.6510406142655083, | |
| "grad_norm": 4.67177152633667, | |
| "learning_rate": 1.6259231275709636e-07, | |
| "logits/chosen": 254.6771697998047, | |
| "logits/rejected": 254.81906127929688, | |
| "logps/chosen": -1.1654198169708252, | |
| "logps/rejected": -1.331291913986206, | |
| "loss": 1.2329, | |
| "odds_ratio_loss": 0.6746650338172913, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.11654196679592133, | |
| "rewards/margins": 0.016587218269705772, | |
| "rewards/rejected": -0.13312919437885284, | |
| "sft_loss": 1.1654198169708252, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.66720549605981, | |
| "grad_norm": 2.415234327316284, | |
| "learning_rate": 1.478999832738548e-07, | |
| "logits/chosen": 253.84658813476562, | |
| "logits/rejected": 254.2846221923828, | |
| "logps/chosen": -1.1557317972183228, | |
| "logps/rejected": -1.4208415746688843, | |
| "loss": 1.2199, | |
| "odds_ratio_loss": 0.6416669487953186, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.11557319015264511, | |
| "rewards/margins": 0.026510965079069138, | |
| "rewards/rejected": -0.14208415150642395, | |
| "sft_loss": 1.1557317972183228, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.683370377854112, | |
| "grad_norm": 2.4805846214294434, | |
| "learning_rate": 1.338830716323769e-07, | |
| "logits/chosen": 253.04605102539062, | |
| "logits/rejected": 253.3199005126953, | |
| "logps/chosen": -1.1211105585098267, | |
| "logps/rejected": -1.3149446249008179, | |
| "loss": 1.1846, | |
| "odds_ratio_loss": 0.6351101994514465, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.11211104691028595, | |
| "rewards/margins": 0.019383419305086136, | |
| "rewards/rejected": -0.13149447739124298, | |
| "sft_loss": 1.1211105585098267, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.699535259648414, | |
| "grad_norm": 4.832085609436035, | |
| "learning_rate": 1.205456028622723e-07, | |
| "logits/chosen": 254.3978729248047, | |
| "logits/rejected": 254.5135498046875, | |
| "logps/chosen": -1.0987465381622314, | |
| "logps/rejected": -1.4824903011322021, | |
| "loss": 1.157, | |
| "odds_ratio_loss": 0.582126259803772, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.1098746508359909, | |
| "rewards/margins": 0.038374386727809906, | |
| "rewards/rejected": -0.1482490599155426, | |
| "sft_loss": 1.0987465381622314, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.7157001414427158, | |
| "grad_norm": 1.8406291007995605, | |
| "learning_rate": 1.0789140688756805e-07, | |
| "logits/chosen": 254.9768524169922, | |
| "logits/rejected": 255.4145965576172, | |
| "logps/chosen": -1.1487500667572021, | |
| "logps/rejected": -1.416771650314331, | |
| "loss": 1.2068, | |
| "odds_ratio_loss": 0.5807241201400757, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.11487500369548798, | |
| "rewards/margins": 0.026802152395248413, | |
| "rewards/rejected": -0.14167717099189758, | |
| "sft_loss": 1.1487500667572021, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.7318650232370176, | |
| "grad_norm": 9.125133514404297, | |
| "learning_rate": 9.592411742693098e-08, | |
| "logits/chosen": 253.77490234375, | |
| "logits/rejected": 253.99368286132812, | |
| "logps/chosen": -1.2172834873199463, | |
| "logps/rejected": -1.371734857559204, | |
| "loss": 1.2881, | |
| "odds_ratio_loss": 0.7082632780075073, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12172834575176239, | |
| "rewards/margins": 0.015445133671164513, | |
| "rewards/rejected": -0.13717348873615265, | |
| "sft_loss": 1.2172834873199463, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 2.7480299050313195, | |
| "grad_norm": 1.7780921459197998, | |
| "learning_rate": 8.464717095022168e-08, | |
| "logits/chosen": 251.9024658203125, | |
| "logits/rejected": 253.18222045898438, | |
| "logps/chosen": -1.1575608253479004, | |
| "logps/rejected": -1.4255679845809937, | |
| "loss": 1.2219, | |
| "odds_ratio_loss": 0.6432427167892456, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.115756094455719, | |
| "rewards/margins": 0.026800716295838356, | |
| "rewards/rejected": -0.1425568014383316, | |
| "sft_loss": 1.1575608253479004, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.7641947868256214, | |
| "grad_norm": 2.707679271697998, | |
| "learning_rate": 7.406380569169841e-08, | |
| "logits/chosen": 254.63046264648438, | |
| "logits/rejected": 255.2324676513672, | |
| "logps/chosen": -1.21084725856781, | |
| "logps/rejected": -1.339179277420044, | |
| "loss": 1.278, | |
| "odds_ratio_loss": 0.6712638139724731, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12108473479747772, | |
| "rewards/margins": 0.012833192944526672, | |
| "rewards/rejected": -0.1339179426431656, | |
| "sft_loss": 1.21084725856781, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 2.7803596686199232, | |
| "grad_norm": 9.321467399597168, | |
| "learning_rate": 6.417706072013808e-08, | |
| "logits/chosen": 255.14419555664062, | |
| "logits/rejected": 255.5975341796875, | |
| "logps/chosen": -1.1439273357391357, | |
| "logps/rejected": -1.3444888591766357, | |
| "loss": 1.2084, | |
| "odds_ratio_loss": 0.644874095916748, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1143927350640297, | |
| "rewards/margins": 0.020056165754795074, | |
| "rewards/rejected": -0.13444891571998596, | |
| "sft_loss": 1.1439273357391357, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 2.796524550414225, | |
| "grad_norm": 4.651902675628662, | |
| "learning_rate": 5.498977506615294e-08, | |
| "logits/chosen": 254.2846221923828, | |
| "logits/rejected": 255.0900421142578, | |
| "logps/chosen": -1.2078436613082886, | |
| "logps/rejected": -1.3565863370895386, | |
| "loss": 1.2795, | |
| "odds_ratio_loss": 0.7164067029953003, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12078437954187393, | |
| "rewards/margins": 0.014874264597892761, | |
| "rewards/rejected": -0.1356586515903473, | |
| "sft_loss": 1.2078436613082886, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 2.812689432208527, | |
| "grad_norm": 2.109281301498413, | |
| "learning_rate": 4.6504586906947756e-08, | |
| "logits/chosen": 255.70327758789062, | |
| "logits/rejected": 255.75326538085938, | |
| "logps/chosen": -1.204192876815796, | |
| "logps/rejected": -1.3763076066970825, | |
| "loss": 1.2661, | |
| "odds_ratio_loss": 0.6194810271263123, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12041930109262466, | |
| "rewards/margins": 0.01721145212650299, | |
| "rewards/rejected": -0.13763076066970825, | |
| "sft_loss": 1.204192876815796, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 2.828854314002829, | |
| "grad_norm": 11.28996467590332, | |
| "learning_rate": 3.8723932808754914e-08, | |
| "logits/chosen": 254.12826538085938, | |
| "logits/rejected": 254.1260528564453, | |
| "logps/chosen": -1.2869278192520142, | |
| "logps/rejected": -1.4129821062088013, | |
| "loss": 1.3563, | |
| "odds_ratio_loss": 0.6939128637313843, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12869277596473694, | |
| "rewards/margins": 0.012605440802872181, | |
| "rewards/rejected": -0.14129820466041565, | |
| "sft_loss": 1.2869278192520142, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.8450191957971307, | |
| "grad_norm": 4.297213077545166, | |
| "learning_rate": 3.1650047027158014e-08, | |
| "logits/chosen": 254.026123046875, | |
| "logits/rejected": 254.0368194580078, | |
| "logps/chosen": -1.1498368978500366, | |
| "logps/rejected": -1.3505172729492188, | |
| "loss": 1.212, | |
| "odds_ratio_loss": 0.6218072175979614, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.11498367786407471, | |
| "rewards/margins": 0.020068055018782616, | |
| "rewards/rejected": -0.13505175709724426, | |
| "sft_loss": 1.1498368978500366, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 2.8611840775914326, | |
| "grad_norm": 3.5106639862060547, | |
| "learning_rate": 2.5284960865517848e-08, | |
| "logits/chosen": 253.19677734375, | |
| "logits/rejected": 253.49380493164062, | |
| "logps/chosen": -1.0654290914535522, | |
| "logps/rejected": -1.4208238124847412, | |
| "loss": 1.1222, | |
| "odds_ratio_loss": 0.5680567026138306, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.1065429076552391, | |
| "rewards/margins": 0.035539474338293076, | |
| "rewards/rejected": -0.14208237826824188, | |
| "sft_loss": 1.0654290914535522, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 2.8773489593857344, | |
| "grad_norm": 1.9635688066482544, | |
| "learning_rate": 1.9630502091670388e-08, | |
| "logits/chosen": 254.31576538085938, | |
| "logits/rejected": 254.73184204101562, | |
| "logps/chosen": -1.16495680809021, | |
| "logps/rejected": -1.438716173171997, | |
| "loss": 1.2222, | |
| "odds_ratio_loss": 0.5726233720779419, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.11649568378925323, | |
| "rewards/margins": 0.027375921607017517, | |
| "rewards/rejected": -0.14387162029743195, | |
| "sft_loss": 1.16495680809021, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 2.8935138411800363, | |
| "grad_norm": 7.4126458168029785, | |
| "learning_rate": 1.4688294413074677e-08, | |
| "logits/chosen": 253.7817840576172, | |
| "logits/rejected": 254.4253692626953, | |
| "logps/chosen": -1.081469178199768, | |
| "logps/rejected": -1.3936357498168945, | |
| "loss": 1.1414, | |
| "odds_ratio_loss": 0.5996376872062683, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.10814690589904785, | |
| "rewards/margins": 0.03121664747595787, | |
| "rewards/rejected": -0.13936355710029602, | |
| "sft_loss": 1.081469178199768, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 2.909678722974338, | |
| "grad_norm": 1.8516889810562134, | |
| "learning_rate": 1.0459757010556626e-08, | |
| "logits/chosen": 252.57345581054688, | |
| "logits/rejected": 252.3190155029297, | |
| "logps/chosen": -1.173514723777771, | |
| "logps/rejected": -1.303662657737732, | |
| "loss": 1.2401, | |
| "odds_ratio_loss": 0.6658231019973755, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.1173514723777771, | |
| "rewards/margins": 0.013014810159802437, | |
| "rewards/rejected": -0.13036629557609558, | |
| "sft_loss": 1.173514723777771, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 2.92584360476864, | |
| "grad_norm": 7.74896764755249, | |
| "learning_rate": 6.94610413078306e-09, | |
| "logits/chosen": 253.01602172851562, | |
| "logits/rejected": 253.71841430664062, | |
| "logps/chosen": -1.2020865678787231, | |
| "logps/rejected": -1.5033478736877441, | |
| "loss": 1.2671, | |
| "odds_ratio_loss": 0.6497219204902649, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12020864337682724, | |
| "rewards/margins": 0.030126124620437622, | |
| "rewards/rejected": -0.15033479034900665, | |
| "sft_loss": 1.2020865678787231, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 2.942008486562942, | |
| "grad_norm": 2.0265376567840576, | |
| "learning_rate": 4.14834473758563e-09, | |
| "logits/chosen": 252.50656127929688, | |
| "logits/rejected": 252.5145721435547, | |
| "logps/chosen": -1.0974245071411133, | |
| "logps/rejected": -1.4048916101455688, | |
| "loss": 1.1558, | |
| "odds_ratio_loss": 0.5838974714279175, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.10974244773387909, | |
| "rewards/margins": 0.030746713280677795, | |
| "rewards/rejected": -0.14048916101455688, | |
| "sft_loss": 1.0974245071411133, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 2.9581733683572438, | |
| "grad_norm": 2.291332721710205, | |
| "learning_rate": 2.067282222230349e-09, | |
| "logits/chosen": 254.31192016601562, | |
| "logits/rejected": 254.5823974609375, | |
| "logps/chosen": -1.1228351593017578, | |
| "logps/rejected": -1.4580986499786377, | |
| "loss": 1.1841, | |
| "odds_ratio_loss": 0.6124657392501831, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.11228351294994354, | |
| "rewards/margins": 0.03352636098861694, | |
| "rewards/rejected": -0.1458098590373993, | |
| "sft_loss": 1.1228351593017578, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 2.9743382501515456, | |
| "grad_norm": 8.377201080322266, | |
| "learning_rate": 7.035141727212979e-10, | |
| "logits/chosen": 252.5790252685547, | |
| "logits/rejected": 253.33993530273438, | |
| "logps/chosen": -1.062239646911621, | |
| "logps/rejected": -1.3180148601531982, | |
| "loss": 1.1223, | |
| "odds_ratio_loss": 0.6001896858215332, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.10622396320104599, | |
| "rewards/margins": 0.025577524676918983, | |
| "rewards/rejected": -0.13180148601531982, | |
| "sft_loss": 1.062239646911621, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 2.9905031319458475, | |
| "grad_norm": 5.447793006896973, | |
| "learning_rate": 5.743220219761592e-11, | |
| "logits/chosen": 254.24691772460938, | |
| "logits/rejected": 254.78369140625, | |
| "logps/chosen": -1.195462942123413, | |
| "logps/rejected": -1.4066945314407349, | |
| "loss": 1.2627, | |
| "odds_ratio_loss": 0.6723325252532959, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.1195463091135025, | |
| "rewards/margins": 0.021123168990015984, | |
| "rewards/rejected": -0.14066946506500244, | |
| "sft_loss": 1.195462942123413, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 2.9969690846635686, | |
| "step": 1854, | |
| "total_flos": 2.1935611788745114e+18, | |
| "train_loss": 1.3469306265266197, | |
| "train_runtime": 24131.5713, | |
| "train_samples_per_second": 1.231, | |
| "train_steps_per_second": 0.077 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1854, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 2.1935611788745114e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |