{ "best_metric": 0.22929497063159943, "best_model_checkpoint": "/scratch/skscla001/results/mms-1b-all-bem-genbed-combined-adapter-test/checkpoint-4800", "epoch": 7.142857142857143, "eval_steps": 100, "global_step": 5200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.13736263736263737, "grad_norm": 3.0789577960968018, "learning_rate": 0.00028799999999999995, "loss": 6.4783, "step": 100 }, { "epoch": 0.13736263736263737, "eval_loss": 0.6402517557144165, "eval_runtime": 132.5796, "eval_samples_per_second": 14.625, "eval_steps_per_second": 1.833, "eval_wer": 0.69, "step": 100 }, { "epoch": 0.27472527472527475, "grad_norm": 1.5154528617858887, "learning_rate": 0.0002986752529898804, "loss": 0.5413, "step": 200 }, { "epoch": 0.27472527472527475, "eval_loss": 0.32597342133522034, "eval_runtime": 131.6167, "eval_samples_per_second": 14.732, "eval_steps_per_second": 1.846, "eval_wer": 0.4931351351351351, "step": 200 }, { "epoch": 0.41208791208791207, "grad_norm": 1.3768789768218994, "learning_rate": 0.0002972953081876725, "loss": 0.437, "step": 300 }, { "epoch": 0.41208791208791207, "eval_loss": 0.30144402384757996, "eval_runtime": 132.6277, "eval_samples_per_second": 14.62, "eval_steps_per_second": 1.832, "eval_wer": 0.46778378378378377, "step": 300 }, { "epoch": 0.5494505494505495, "grad_norm": 2.0690324306488037, "learning_rate": 0.00029591536338546457, "loss": 0.4402, "step": 400 }, { "epoch": 0.5494505494505495, "eval_loss": 0.29820021986961365, "eval_runtime": 132.486, "eval_samples_per_second": 14.636, "eval_steps_per_second": 1.834, "eval_wer": 0.4817837837837838, "step": 400 }, { "epoch": 0.6868131868131868, "grad_norm": 1.9618090391159058, "learning_rate": 0.0002945354185832566, "loss": 0.4153, "step": 500 }, { "epoch": 0.6868131868131868, "eval_loss": 0.2935521602630615, "eval_runtime": 133.0965, "eval_samples_per_second": 14.568, "eval_steps_per_second": 1.826, "eval_wer": 0.47021621621621623, "step": 500 }, { "epoch": 0.8241758241758241, "grad_norm": 1.3698606491088867, "learning_rate": 0.0002931554737810487, "loss": 0.4154, "step": 600 }, { "epoch": 0.8241758241758241, "eval_loss": 0.2883508503437042, "eval_runtime": 131.9274, "eval_samples_per_second": 14.697, "eval_steps_per_second": 1.842, "eval_wer": 0.4492972972972973, "step": 600 }, { "epoch": 0.9615384615384616, "grad_norm": 1.5283702611923218, "learning_rate": 0.0002917755289788408, "loss": 0.3789, "step": 700 }, { "epoch": 0.9615384615384616, "eval_loss": 0.2806306481361389, "eval_runtime": 132.4493, "eval_samples_per_second": 14.64, "eval_steps_per_second": 1.835, "eval_wer": 0.4521081081081081, "step": 700 }, { "epoch": 1.098901098901099, "grad_norm": 0.4610430598258972, "learning_rate": 0.0002903955841766329, "loss": 0.3667, "step": 800 }, { "epoch": 1.098901098901099, "eval_loss": 0.2763822078704834, "eval_runtime": 133.2255, "eval_samples_per_second": 14.554, "eval_steps_per_second": 1.824, "eval_wer": 0.4351891891891892, "step": 800 }, { "epoch": 1.2362637362637363, "grad_norm": 0.7730151414871216, "learning_rate": 0.000289015639374425, "loss": 0.3929, "step": 900 }, { "epoch": 1.2362637362637363, "eval_loss": 0.27625423669815063, "eval_runtime": 133.7811, "eval_samples_per_second": 14.494, "eval_steps_per_second": 1.816, "eval_wer": 0.46535135135135136, "step": 900 }, { "epoch": 1.3736263736263736, "grad_norm": 0.9868985414505005, "learning_rate": 0.0002876356945722171, "loss": 0.3847, "step": 1000 }, { "epoch": 1.3736263736263736, "eval_loss": 0.27053016424179077, "eval_runtime": 133.1254, "eval_samples_per_second": 14.565, "eval_steps_per_second": 1.825, "eval_wer": 0.44064864864864867, "step": 1000 }, { "epoch": 1.510989010989011, "grad_norm": 0.7302649021148682, "learning_rate": 0.00028625574977000917, "loss": 0.3833, "step": 1100 }, { "epoch": 1.510989010989011, "eval_loss": 0.2696720063686371, "eval_runtime": 134.0459, "eval_samples_per_second": 14.465, "eval_steps_per_second": 1.813, "eval_wer": 0.42464864864864865, "step": 1100 }, { "epoch": 1.6483516483516483, "grad_norm": 0.8602890372276306, "learning_rate": 0.00028487580496780126, "loss": 0.3742, "step": 1200 }, { "epoch": 1.6483516483516483, "eval_loss": 0.2667711675167084, "eval_runtime": 133.3019, "eval_samples_per_second": 14.546, "eval_steps_per_second": 1.823, "eval_wer": 0.42502702702702705, "step": 1200 }, { "epoch": 1.7857142857142856, "grad_norm": 1.5730996131896973, "learning_rate": 0.00028349586016559336, "loss": 0.3694, "step": 1300 }, { "epoch": 1.7857142857142856, "eval_loss": 0.2689613401889801, "eval_runtime": 133.41, "eval_samples_per_second": 14.534, "eval_steps_per_second": 1.821, "eval_wer": 0.4189189189189189, "step": 1300 }, { "epoch": 1.9230769230769231, "grad_norm": 1.534987449645996, "learning_rate": 0.00028211591536338545, "loss": 0.3494, "step": 1400 }, { "epoch": 1.9230769230769231, "eval_loss": 0.26353445649147034, "eval_runtime": 132.3792, "eval_samples_per_second": 14.647, "eval_steps_per_second": 1.836, "eval_wer": 0.416, "step": 1400 }, { "epoch": 2.0604395604395602, "grad_norm": 1.566632866859436, "learning_rate": 0.00028073597056117754, "loss": 0.3724, "step": 1500 }, { "epoch": 2.0604395604395602, "eval_loss": 0.2625681757926941, "eval_runtime": 134.0736, "eval_samples_per_second": 14.462, "eval_steps_per_second": 1.812, "eval_wer": 0.4323243243243243, "step": 1500 }, { "epoch": 2.197802197802198, "grad_norm": 1.1042734384536743, "learning_rate": 0.00027935602575896964, "loss": 0.3723, "step": 1600 }, { "epoch": 2.197802197802198, "eval_loss": 0.2598220705986023, "eval_runtime": 132.9896, "eval_samples_per_second": 14.58, "eval_steps_per_second": 1.827, "eval_wer": 0.4247027027027027, "step": 1600 }, { "epoch": 2.3351648351648353, "grad_norm": 1.444578766822815, "learning_rate": 0.00027797608095676173, "loss": 0.3505, "step": 1700 }, { "epoch": 2.3351648351648353, "eval_loss": 0.25828292965888977, "eval_runtime": 133.6765, "eval_samples_per_second": 14.505, "eval_steps_per_second": 1.818, "eval_wer": 0.412, "step": 1700 }, { "epoch": 2.4725274725274726, "grad_norm": 2.0142714977264404, "learning_rate": 0.00027659613615455377, "loss": 0.3393, "step": 1800 }, { "epoch": 2.4725274725274726, "eval_loss": 0.2563156187534332, "eval_runtime": 132.7739, "eval_samples_per_second": 14.604, "eval_steps_per_second": 1.83, "eval_wer": 0.41281081081081084, "step": 1800 }, { "epoch": 2.60989010989011, "grad_norm": 2.562643527984619, "learning_rate": 0.00027521619135234586, "loss": 0.3352, "step": 1900 }, { "epoch": 2.60989010989011, "eval_loss": 0.25453999638557434, "eval_runtime": 133.7423, "eval_samples_per_second": 14.498, "eval_steps_per_second": 1.817, "eval_wer": 0.4155675675675676, "step": 1900 }, { "epoch": 2.7472527472527473, "grad_norm": 3.292581796646118, "learning_rate": 0.00027383624655013796, "loss": 0.3516, "step": 2000 }, { "epoch": 2.7472527472527473, "eval_loss": 0.25513678789138794, "eval_runtime": 133.1058, "eval_samples_per_second": 14.567, "eval_steps_per_second": 1.826, "eval_wer": 0.4315135135135135, "step": 2000 }, { "epoch": 2.8846153846153846, "grad_norm": 0.9819747805595398, "learning_rate": 0.00027245630174793005, "loss": 0.3489, "step": 2100 }, { "epoch": 2.8846153846153846, "eval_loss": 0.256041020154953, "eval_runtime": 134.2008, "eval_samples_per_second": 14.448, "eval_steps_per_second": 1.811, "eval_wer": 0.426972972972973, "step": 2100 }, { "epoch": 3.021978021978022, "grad_norm": 0.77928626537323, "learning_rate": 0.00027107635694572214, "loss": 0.3512, "step": 2200 }, { "epoch": 3.021978021978022, "eval_loss": 0.25362125039100647, "eval_runtime": 132.6388, "eval_samples_per_second": 14.619, "eval_steps_per_second": 1.832, "eval_wer": 0.4038918918918919, "step": 2200 }, { "epoch": 3.159340659340659, "grad_norm": 1.5085543394088745, "learning_rate": 0.00026971021159153633, "loss": 0.339, "step": 2300 }, { "epoch": 3.159340659340659, "eval_loss": 0.24898956716060638, "eval_runtime": 133.1617, "eval_samples_per_second": 14.561, "eval_steps_per_second": 1.825, "eval_wer": 0.3988648648648649, "step": 2300 }, { "epoch": 3.2967032967032965, "grad_norm": 0.6166655421257019, "learning_rate": 0.0002683302667893284, "loss": 0.3374, "step": 2400 }, { "epoch": 3.2967032967032965, "eval_loss": 0.24945656955242157, "eval_runtime": 132.9861, "eval_samples_per_second": 14.58, "eval_steps_per_second": 1.827, "eval_wer": 0.39637837837837836, "step": 2400 }, { "epoch": 3.4340659340659343, "grad_norm": 0.9863089919090271, "learning_rate": 0.00026695032198712046, "loss": 0.3295, "step": 2500 }, { "epoch": 3.4340659340659343, "eval_loss": 0.2518012523651123, "eval_runtime": 133.4804, "eval_samples_per_second": 14.526, "eval_steps_per_second": 1.82, "eval_wer": 0.4037297297297297, "step": 2500 }, { "epoch": 3.571428571428571, "grad_norm": 0.5508905649185181, "learning_rate": 0.00026557037718491256, "loss": 0.3391, "step": 2600 }, { "epoch": 3.571428571428571, "eval_loss": 0.24910376965999603, "eval_runtime": 133.409, "eval_samples_per_second": 14.534, "eval_steps_per_second": 1.821, "eval_wer": 0.4077297297297297, "step": 2600 }, { "epoch": 3.708791208791209, "grad_norm": 0.5989585518836975, "learning_rate": 0.00026419043238270465, "loss": 0.3373, "step": 2700 }, { "epoch": 3.708791208791209, "eval_loss": 0.24448369443416595, "eval_runtime": 133.5414, "eval_samples_per_second": 14.52, "eval_steps_per_second": 1.82, "eval_wer": 0.3989189189189189, "step": 2700 }, { "epoch": 3.8461538461538463, "grad_norm": 0.696869969367981, "learning_rate": 0.00026281048758049674, "loss": 0.3097, "step": 2800 }, { "epoch": 3.8461538461538463, "eval_loss": 0.24623404443264008, "eval_runtime": 134.7186, "eval_samples_per_second": 14.393, "eval_steps_per_second": 1.804, "eval_wer": 0.4117837837837838, "step": 2800 }, { "epoch": 3.9835164835164836, "grad_norm": 0.8349074721336365, "learning_rate": 0.00026143054277828884, "loss": 0.3458, "step": 2900 }, { "epoch": 3.9835164835164836, "eval_loss": 0.24432998895645142, "eval_runtime": 133.9009, "eval_samples_per_second": 14.481, "eval_steps_per_second": 1.815, "eval_wer": 0.4034054054054054, "step": 2900 }, { "epoch": 4.1208791208791204, "grad_norm": 0.9415439963340759, "learning_rate": 0.00026005059797608093, "loss": 0.313, "step": 3000 }, { "epoch": 4.1208791208791204, "eval_loss": 0.24325776100158691, "eval_runtime": 133.6886, "eval_samples_per_second": 14.504, "eval_steps_per_second": 1.818, "eval_wer": 0.3882162162162162, "step": 3000 }, { "epoch": 4.258241758241758, "grad_norm": 1.5970408916473389, "learning_rate": 0.000258670653173873, "loss": 0.3171, "step": 3100 }, { "epoch": 4.258241758241758, "eval_loss": 0.24258361756801605, "eval_runtime": 135.0574, "eval_samples_per_second": 14.357, "eval_steps_per_second": 1.799, "eval_wer": 0.3968108108108108, "step": 3100 }, { "epoch": 4.395604395604396, "grad_norm": 1.8427647352218628, "learning_rate": 0.0002572907083716651, "loss": 0.3122, "step": 3200 }, { "epoch": 4.395604395604396, "eval_loss": 0.2430083155632019, "eval_runtime": 133.0857, "eval_samples_per_second": 14.57, "eval_steps_per_second": 1.826, "eval_wer": 0.39356756756756756, "step": 3200 }, { "epoch": 4.532967032967033, "grad_norm": 0.6071587800979614, "learning_rate": 0.0002559107635694572, "loss": 0.3255, "step": 3300 }, { "epoch": 4.532967032967033, "eval_loss": 0.24039362370967865, "eval_runtime": 133.7381, "eval_samples_per_second": 14.498, "eval_steps_per_second": 1.817, "eval_wer": 0.38216216216216214, "step": 3300 }, { "epoch": 4.670329670329671, "grad_norm": 0.9580681324005127, "learning_rate": 0.0002545308187672493, "loss": 0.3253, "step": 3400 }, { "epoch": 4.670329670329671, "eval_loss": 0.2355840504169464, "eval_runtime": 132.9989, "eval_samples_per_second": 14.579, "eval_steps_per_second": 1.827, "eval_wer": 0.3945945945945946, "step": 3400 }, { "epoch": 4.8076923076923075, "grad_norm": 1.4110567569732666, "learning_rate": 0.0002531508739650414, "loss": 0.3341, "step": 3500 }, { "epoch": 4.8076923076923075, "eval_loss": 0.23690326511859894, "eval_runtime": 133.2507, "eval_samples_per_second": 14.552, "eval_steps_per_second": 1.824, "eval_wer": 0.3872432432432432, "step": 3500 }, { "epoch": 4.945054945054945, "grad_norm": 1.9591115713119507, "learning_rate": 0.0002517709291628335, "loss": 0.3183, "step": 3600 }, { "epoch": 4.945054945054945, "eval_loss": 0.2345106154680252, "eval_runtime": 133.4011, "eval_samples_per_second": 14.535, "eval_steps_per_second": 1.822, "eval_wer": 0.38535135135135135, "step": 3600 }, { "epoch": 5.082417582417582, "grad_norm": 0.616919219493866, "learning_rate": 0.0002503909843606256, "loss": 0.3461, "step": 3700 }, { "epoch": 5.082417582417582, "eval_loss": 0.23949113488197327, "eval_runtime": 133.8985, "eval_samples_per_second": 14.481, "eval_steps_per_second": 1.815, "eval_wer": 0.3828108108108108, "step": 3700 }, { "epoch": 5.21978021978022, "grad_norm": 0.7555241584777832, "learning_rate": 0.0002490110395584176, "loss": 0.3147, "step": 3800 }, { "epoch": 5.21978021978022, "eval_loss": 0.23591694235801697, "eval_runtime": 133.3437, "eval_samples_per_second": 14.541, "eval_steps_per_second": 1.822, "eval_wer": 0.37751351351351353, "step": 3800 }, { "epoch": 5.357142857142857, "grad_norm": 0.6169973015785217, "learning_rate": 0.0002476310947562097, "loss": 0.317, "step": 3900 }, { "epoch": 5.357142857142857, "eval_loss": 0.2320287972688675, "eval_runtime": 134.295, "eval_samples_per_second": 14.438, "eval_steps_per_second": 1.809, "eval_wer": 0.38075675675675674, "step": 3900 }, { "epoch": 5.4945054945054945, "grad_norm": 0.44300732016563416, "learning_rate": 0.0002462511499540018, "loss": 0.3094, "step": 4000 }, { "epoch": 5.4945054945054945, "eval_loss": 0.23655937612056732, "eval_runtime": 134.695, "eval_samples_per_second": 14.395, "eval_steps_per_second": 1.804, "eval_wer": 0.37972972972972974, "step": 4000 }, { "epoch": 5.631868131868131, "grad_norm": 1.5346554517745972, "learning_rate": 0.0002448712051517939, "loss": 0.2913, "step": 4100 }, { "epoch": 5.631868131868131, "eval_loss": 0.2357349544763565, "eval_runtime": 134.0434, "eval_samples_per_second": 14.465, "eval_steps_per_second": 1.813, "eval_wer": 0.37491891891891893, "step": 4100 }, { "epoch": 5.769230769230769, "grad_norm": 0.7017680406570435, "learning_rate": 0.000243491260349586, "loss": 0.3195, "step": 4200 }, { "epoch": 5.769230769230769, "eval_loss": 0.23318681120872498, "eval_runtime": 133.1069, "eval_samples_per_second": 14.567, "eval_steps_per_second": 1.826, "eval_wer": 0.3694054054054054, "step": 4200 }, { "epoch": 5.906593406593407, "grad_norm": 0.46099045872688293, "learning_rate": 0.00024212511499540016, "loss": 0.3189, "step": 4300 }, { "epoch": 5.906593406593407, "eval_loss": 0.23127850890159607, "eval_runtime": 135.2118, "eval_samples_per_second": 14.34, "eval_steps_per_second": 1.797, "eval_wer": 0.38697297297297295, "step": 4300 }, { "epoch": 6.043956043956044, "grad_norm": 1.010475516319275, "learning_rate": 0.00024074517019319225, "loss": 0.3105, "step": 4400 }, { "epoch": 6.043956043956044, "eval_loss": 0.23260627686977386, "eval_runtime": 134.6368, "eval_samples_per_second": 14.402, "eval_steps_per_second": 1.805, "eval_wer": 0.38064864864864867, "step": 4400 }, { "epoch": 6.181318681318682, "grad_norm": 0.7957504391670227, "learning_rate": 0.00023936522539098434, "loss": 0.2937, "step": 4500 }, { "epoch": 6.181318681318682, "eval_loss": 0.23464266955852509, "eval_runtime": 134.508, "eval_samples_per_second": 14.415, "eval_steps_per_second": 1.807, "eval_wer": 0.3783783783783784, "step": 4500 }, { "epoch": 6.318681318681318, "grad_norm": 0.73511803150177, "learning_rate": 0.00023798528058877644, "loss": 0.3088, "step": 4600 }, { "epoch": 6.318681318681318, "eval_loss": 0.2313350886106491, "eval_runtime": 135.48, "eval_samples_per_second": 14.312, "eval_steps_per_second": 1.794, "eval_wer": 0.3725945945945946, "step": 4600 }, { "epoch": 6.456043956043956, "grad_norm": 0.587340772151947, "learning_rate": 0.00023660533578656853, "loss": 0.2852, "step": 4700 }, { "epoch": 6.456043956043956, "eval_loss": 0.2307436466217041, "eval_runtime": 135.4353, "eval_samples_per_second": 14.317, "eval_steps_per_second": 1.794, "eval_wer": 0.37086486486486486, "step": 4700 }, { "epoch": 6.593406593406593, "grad_norm": 0.5956315994262695, "learning_rate": 0.0002352253909843606, "loss": 0.3083, "step": 4800 }, { "epoch": 6.593406593406593, "eval_loss": 0.22929497063159943, "eval_runtime": 134.2045, "eval_samples_per_second": 14.448, "eval_steps_per_second": 1.811, "eval_wer": 0.3751891891891892, "step": 4800 }, { "epoch": 6.730769230769231, "grad_norm": 1.3765567541122437, "learning_rate": 0.0002338454461821527, "loss": 0.3194, "step": 4900 }, { "epoch": 6.730769230769231, "eval_loss": 0.22924266755580902, "eval_runtime": 135.6964, "eval_samples_per_second": 14.289, "eval_steps_per_second": 1.791, "eval_wer": 0.37108108108108107, "step": 4900 }, { "epoch": 6.868131868131869, "grad_norm": 0.7330417037010193, "learning_rate": 0.00023246550137994478, "loss": 0.297, "step": 5000 }, { "epoch": 6.868131868131869, "eval_loss": 0.2303166538476944, "eval_runtime": 134.3817, "eval_samples_per_second": 14.429, "eval_steps_per_second": 1.808, "eval_wer": 0.3715135135135135, "step": 5000 }, { "epoch": 7.0054945054945055, "grad_norm": 0.9262694716453552, "learning_rate": 0.00023108555657773688, "loss": 0.3086, "step": 5100 }, { "epoch": 7.0054945054945055, "eval_loss": 0.23395021259784698, "eval_runtime": 134.6683, "eval_samples_per_second": 14.398, "eval_steps_per_second": 1.804, "eval_wer": 0.4027027027027027, "step": 5100 }, { "epoch": 7.142857142857143, "grad_norm": 0.5672425627708435, "learning_rate": 0.00022970561177552897, "loss": 0.3058, "step": 5200 }, { "epoch": 7.142857142857143, "eval_loss": 0.22942779958248138, "eval_runtime": 133.8761, "eval_samples_per_second": 14.484, "eval_steps_per_second": 1.815, "eval_wer": 0.36648648648648646, "step": 5200 }, { "epoch": 7.142857142857143, "step": 5200, "total_flos": 2.4864489394936607e+19, "train_loss": 0.46319753646850587, "train_runtime": 13134.8195, "train_samples_per_second": 13.286, "train_steps_per_second": 1.663 } ], "logging_steps": 100, "max_steps": 21840, "num_input_tokens_seen": 0, "num_train_epochs": 30, "save_steps": 200, "stateful_callbacks": { "EarlyStoppingCallback": { "args": { "early_stopping_patience": 3, "early_stopping_threshold": 0.0 }, "attributes": { "early_stopping_patience_counter": 3 } }, "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 2.4864489394936607e+19, "train_batch_size": 8, "trial_name": null, "trial_params": null }