File size: 13,564 Bytes
7eb44e2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 |
{
"best_metric": 0.40118029713630676,
"best_model_checkpoint": "./mistral/19-04-24-Weni-WeniGPT-Agents-Mistral-1.0.6-SFT-1.0.3-DPO_Experiment on DPO with other hyperparameters and best SFT model of WeniGPT-2_max_steps-366_batch_4_2024-04-19_ppid_9/checkpoint-180",
"epoch": 2.926829268292683,
"eval_steps": 30,
"global_step": 180,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.16,
"grad_norm": 29.063692092895508,
"learning_rate": 3.181818181818182e-06,
"logits/chosen": -1.7455532550811768,
"logits/rejected": -1.7793405055999756,
"logps/chosen": -54.72136306762695,
"logps/rejected": -97.28812408447266,
"loss": 0.6924,
"rewards/accuracies": 0.15000000596046448,
"rewards/chosen": 0.003122234484180808,
"rewards/margins": 0.0015356539515778422,
"rewards/rejected": 0.0015865802997723222,
"step": 10
},
{
"epoch": 0.33,
"grad_norm": 15.157337188720703,
"learning_rate": 4.9154929577464795e-06,
"logits/chosen": -1.8641210794448853,
"logits/rejected": -1.8705828189849854,
"logps/chosen": -40.3618049621582,
"logps/rejected": -55.96257400512695,
"loss": 0.6737,
"rewards/accuracies": 0.32499998807907104,
"rewards/chosen": 0.04275550693273544,
"rewards/margins": 0.040880247950553894,
"rewards/rejected": 0.0018752537434920669,
"step": 20
},
{
"epoch": 0.49,
"grad_norm": 13.148392677307129,
"learning_rate": 4.774647887323944e-06,
"logits/chosen": -1.8284633159637451,
"logits/rejected": -1.8519256114959717,
"logps/chosen": -44.69489288330078,
"logps/rejected": -76.79936981201172,
"loss": 0.6371,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": 0.12410124391317368,
"rewards/margins": 0.1248731017112732,
"rewards/rejected": -0.0007718756678514183,
"step": 30
},
{
"epoch": 0.49,
"eval_logits/chosen": -1.7379764318466187,
"eval_logits/rejected": -1.7726188898086548,
"eval_logps/chosen": -50.49373245239258,
"eval_logps/rejected": -83.54340362548828,
"eval_loss": 0.5864880681037903,
"eval_rewards/accuracies": 0.4642857015132904,
"eval_rewards/chosen": 0.24716782569885254,
"eval_rewards/margins": 0.25581416487693787,
"eval_rewards/rejected": -0.008646338246762753,
"eval_runtime": 8.1282,
"eval_samples_per_second": 3.445,
"eval_steps_per_second": 1.722,
"step": 30
},
{
"epoch": 0.65,
"grad_norm": 24.143388748168945,
"learning_rate": 4.633802816901409e-06,
"logits/chosen": -1.8080145120620728,
"logits/rejected": -1.8319604396820068,
"logps/chosen": -46.848201751708984,
"logps/rejected": -81.42877960205078,
"loss": 0.5649,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": 0.3360167443752289,
"rewards/margins": 0.34645381569862366,
"rewards/rejected": -0.010437069460749626,
"step": 40
},
{
"epoch": 0.81,
"grad_norm": 0.0,
"learning_rate": 4.492957746478874e-06,
"logits/chosen": -1.8638660907745361,
"logits/rejected": -1.8754875659942627,
"logps/chosen": -22.06886863708496,
"logps/rejected": -41.17422103881836,
"loss": 0.6377,
"rewards/accuracies": 0.20000000298023224,
"rewards/chosen": 0.25195080041885376,
"rewards/margins": 0.16642150282859802,
"rewards/rejected": 0.08552931249141693,
"step": 50
},
{
"epoch": 0.98,
"grad_norm": 7.51575231552124,
"learning_rate": 4.352112676056338e-06,
"logits/chosen": -1.8215672969818115,
"logits/rejected": -1.8419084548950195,
"logps/chosen": -44.14409255981445,
"logps/rejected": -77.38871002197266,
"loss": 0.5496,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": 0.4203863739967346,
"rewards/margins": 0.458610475063324,
"rewards/rejected": -0.038224123418331146,
"step": 60
},
{
"epoch": 0.98,
"eval_logits/chosen": -1.7419661283493042,
"eval_logits/rejected": -1.7773857116699219,
"eval_logps/chosen": -49.36268997192383,
"eval_logps/rejected": -83.60606384277344,
"eval_loss": 0.4963766038417816,
"eval_rewards/accuracies": 0.4642857015132904,
"eval_rewards/chosen": 0.5864797234535217,
"eval_rewards/margins": 0.613922655582428,
"eval_rewards/rejected": -0.02744293212890625,
"eval_runtime": 8.1315,
"eval_samples_per_second": 3.443,
"eval_steps_per_second": 1.722,
"step": 60
},
{
"epoch": 1.14,
"grad_norm": 3.454394817352295,
"learning_rate": 4.211267605633803e-06,
"logits/chosen": -1.7844343185424805,
"logits/rejected": -1.8126220703125,
"logps/chosen": -39.737998962402344,
"logps/rejected": -78.76033782958984,
"loss": 0.4925,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": 0.6417733430862427,
"rewards/margins": 0.7573493123054504,
"rewards/rejected": -0.11557593196630478,
"step": 70
},
{
"epoch": 1.3,
"grad_norm": 14.345088005065918,
"learning_rate": 4.070422535211268e-06,
"logits/chosen": -1.822148084640503,
"logits/rejected": -1.8463541269302368,
"logps/chosen": -35.735836029052734,
"logps/rejected": -75.4683837890625,
"loss": 0.4965,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.6439998745918274,
"rewards/margins": 0.8037986755371094,
"rewards/rejected": -0.1597987860441208,
"step": 80
},
{
"epoch": 1.46,
"grad_norm": 12.988151550292969,
"learning_rate": 3.9295774647887325e-06,
"logits/chosen": -1.858496069908142,
"logits/rejected": -1.8683990240097046,
"logps/chosen": -30.166736602783203,
"logps/rejected": -50.970542907714844,
"loss": 0.5185,
"rewards/accuracies": 0.30000001192092896,
"rewards/chosen": 0.6545813679695129,
"rewards/margins": 0.7556509971618652,
"rewards/rejected": -0.10106959193944931,
"step": 90
},
{
"epoch": 1.46,
"eval_logits/chosen": -1.7461166381835938,
"eval_logits/rejected": -1.7827314138412476,
"eval_logps/chosen": -47.9538688659668,
"eval_logps/rejected": -83.84149169921875,
"eval_loss": 0.4401554763317108,
"eval_rewards/accuracies": 0.4642857015132904,
"eval_rewards/chosen": 1.0091263055801392,
"eval_rewards/margins": 1.1071958541870117,
"eval_rewards/rejected": -0.09806957095861435,
"eval_runtime": 8.1286,
"eval_samples_per_second": 3.445,
"eval_steps_per_second": 1.722,
"step": 90
},
{
"epoch": 1.63,
"grad_norm": 7.574762344360352,
"learning_rate": 3.7887323943661976e-06,
"logits/chosen": -1.8498780727386475,
"logits/rejected": -1.8676494359970093,
"logps/chosen": -50.50703811645508,
"logps/rejected": -83.80486297607422,
"loss": 0.4752,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": 1.088062047958374,
"rewards/margins": 1.0784289836883545,
"rewards/rejected": 0.009633284993469715,
"step": 100
},
{
"epoch": 1.79,
"grad_norm": 10.141860008239746,
"learning_rate": 3.6478873239436626e-06,
"logits/chosen": -1.8536640405654907,
"logits/rejected": -1.8669731616973877,
"logps/chosen": -33.00554275512695,
"logps/rejected": -46.077144622802734,
"loss": 0.5439,
"rewards/accuracies": 0.30000001192092896,
"rewards/chosen": 0.7643357515335083,
"rewards/margins": 0.7525759339332581,
"rewards/rejected": 0.011759823188185692,
"step": 110
},
{
"epoch": 1.95,
"grad_norm": 0.0,
"learning_rate": 3.5070422535211268e-06,
"logits/chosen": -1.8447682857513428,
"logits/rejected": -1.8607944250106812,
"logps/chosen": -42.43408966064453,
"logps/rejected": -73.089599609375,
"loss": 0.4623,
"rewards/accuracies": 0.375,
"rewards/chosen": 1.1391878128051758,
"rewards/margins": 1.3124778270721436,
"rewards/rejected": -0.17328998446464539,
"step": 120
},
{
"epoch": 1.95,
"eval_logits/chosen": -1.7511886358261108,
"eval_logits/rejected": -1.7883756160736084,
"eval_logps/chosen": -46.9849739074707,
"eval_logps/rejected": -84.11779022216797,
"eval_loss": 0.4216788709163666,
"eval_rewards/accuracies": 0.4642857015132904,
"eval_rewards/chosen": 1.2997931241989136,
"eval_rewards/margins": 1.4807531833648682,
"eval_rewards/rejected": -0.1809600442647934,
"eval_runtime": 8.133,
"eval_samples_per_second": 3.443,
"eval_steps_per_second": 1.721,
"step": 120
},
{
"epoch": 2.11,
"grad_norm": 0.7550886869430542,
"learning_rate": 3.3661971830985918e-06,
"logits/chosen": -1.765539526939392,
"logits/rejected": -1.8175758123397827,
"logps/chosen": -50.52437973022461,
"logps/rejected": -116.59224700927734,
"loss": 0.3356,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 1.8601114749908447,
"rewards/margins": 2.246537685394287,
"rewards/rejected": -0.3864262104034424,
"step": 130
},
{
"epoch": 2.28,
"grad_norm": 0.23429778218269348,
"learning_rate": 3.225352112676057e-06,
"logits/chosen": -1.7902629375457764,
"logits/rejected": -1.8166630268096924,
"logps/chosen": -41.8301887512207,
"logps/rejected": -78.8285140991211,
"loss": 0.4254,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": 1.3376247882843018,
"rewards/margins": 1.733752965927124,
"rewards/rejected": -0.39612799882888794,
"step": 140
},
{
"epoch": 2.44,
"grad_norm": 0.4628850817680359,
"learning_rate": 3.084507042253521e-06,
"logits/chosen": -1.8713032007217407,
"logits/rejected": -1.8906549215316772,
"logps/chosen": -38.51209259033203,
"logps/rejected": -69.11054992675781,
"loss": 0.4985,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": 1.159945011138916,
"rewards/margins": 1.5182414054870605,
"rewards/rejected": -0.35829633474349976,
"step": 150
},
{
"epoch": 2.44,
"eval_logits/chosen": -1.7590656280517578,
"eval_logits/rejected": -1.7967649698257446,
"eval_logps/chosen": -45.99831008911133,
"eval_logps/rejected": -84.5901107788086,
"eval_loss": 0.40688762068748474,
"eval_rewards/accuracies": 0.4642857015132904,
"eval_rewards/chosen": 1.5957926511764526,
"eval_rewards/margins": 1.9184513092041016,
"eval_rewards/rejected": -0.3226587474346161,
"eval_runtime": 8.13,
"eval_samples_per_second": 3.444,
"eval_steps_per_second": 1.722,
"step": 150
},
{
"epoch": 2.6,
"grad_norm": 2.2907166481018066,
"learning_rate": 2.943661971830986e-06,
"logits/chosen": -1.9065347909927368,
"logits/rejected": -1.9184277057647705,
"logps/chosen": -35.44993209838867,
"logps/rejected": -58.14739227294922,
"loss": 0.5111,
"rewards/accuracies": 0.30000001192092896,
"rewards/chosen": 1.2248560190200806,
"rewards/margins": 1.3947786092758179,
"rewards/rejected": -0.1699226349592209,
"step": 160
},
{
"epoch": 2.76,
"grad_norm": 0.19129106402397156,
"learning_rate": 2.802816901408451e-06,
"logits/chosen": -1.7800159454345703,
"logits/rejected": -1.8162111043930054,
"logps/chosen": -44.746212005615234,
"logps/rejected": -87.70552062988281,
"loss": 0.3875,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": 1.9709749221801758,
"rewards/margins": 2.394925355911255,
"rewards/rejected": -0.42395058274269104,
"step": 170
},
{
"epoch": 2.93,
"grad_norm": 0.09690173715353012,
"learning_rate": 2.6619718309859156e-06,
"logits/chosen": -1.9381850957870483,
"logits/rejected": -1.9563757181167603,
"logps/chosen": -20.314172744750977,
"logps/rejected": -45.916996002197266,
"loss": 0.5276,
"rewards/accuracies": 0.25,
"rewards/chosen": 0.9397296905517578,
"rewards/margins": 1.2702549695968628,
"rewards/rejected": -0.33052533864974976,
"step": 180
},
{
"epoch": 2.93,
"eval_logits/chosen": -1.7637720108032227,
"eval_logits/rejected": -1.8018139600753784,
"eval_logps/chosen": -45.44319534301758,
"eval_logps/rejected": -84.9321517944336,
"eval_loss": 0.40118029713630676,
"eval_rewards/accuracies": 0.4642857015132904,
"eval_rewards/chosen": 1.762328028678894,
"eval_rewards/margins": 2.1875970363616943,
"eval_rewards/rejected": -0.425269216299057,
"eval_runtime": 8.1311,
"eval_samples_per_second": 3.444,
"eval_steps_per_second": 1.722,
"step": 180
}
],
"logging_steps": 10,
"max_steps": 366,
"num_input_tokens_seen": 0,
"num_train_epochs": 6,
"save_steps": 90,
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}
|