|
> Training Environment: |
|
| > Backend: Torch |
|
| > Mixed precision: False |
|
| > Precision: float32 |
|
| > Current device: 0 |
|
| > Num. of GPUs: 1 |
|
| > Num. of CPUs: 4 |
|
| > Num. of Torch Threads: 1 |
|
| > Torch seed: 54321 |
|
| > Torch CUDNN: True |
|
| > Torch CUDNN deterministic: False |
|
| > Torch CUDNN benchmark: False |
|
| > Torch TF32 MatMul: False |
|
> Start Tensorboard: tensorboard --logdir=/kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804 |
|
|
|
> Model has 522165080 parameters |
|
|
|
[4m[1m > EPOCH: 0/3[0m |
|
--> /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804 |
|
|
|
[1m > TRAINING (2024-10-26 07:55:23) [0m |
|
|
|
[1m --> TIME: 2024-10-26 07:55:28 -- STEP: 0/30063 -- GLOBAL_STEP: 0[0m |
|
| > loss_text_ce: 0.23170489072799683 (0.23170489072799683) |
|
| > loss_mel_ce: 6.881418704986572 (6.881418704986572) |
|
| > loss: 1.7782808542251587 (1.7782808542251587) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.8919 (0.8918666839599609) |
|
| > loader_time: 3.9073 (3.9073474407196045) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:55:55 -- STEP: 50/30063 -- GLOBAL_STEP: 50[0m |
|
| > loss_text_ce: 0.21212132275104523 (0.22003370970487596) |
|
| > loss_mel_ce: 6.219595909118652 (6.497453689575195) |
|
| > loss: 1.6079293489456177 (1.6793718528747559) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3578 (0.321729998588562) |
|
| > loader_time: 0.0161 (0.014391927719116212) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:56:21 -- STEP: 100/30063 -- GLOBAL_STEP: 100[0m |
|
| > loss_text_ce: 0.2021128386259079 (0.21027226582169534) |
|
| > loss_mel_ce: 6.183087348937988 (6.344042897224426) |
|
| > loss: 1.5963000059127808 (1.6385787904262543) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2329 (0.3162407755851746) |
|
| > loader_time: 0.0091 (0.013851547241210937) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:56:48 -- STEP: 150/30063 -- GLOBAL_STEP: 150[0m |
|
| > loss_text_ce: 0.17603746056556702 (0.20275207271178564) |
|
| > loss_mel_ce: 6.267411231994629 (6.232032321294149) |
|
| > loss: 1.610862135887146 (1.6086960975329079) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3582 (0.3162189785639445) |
|
| > loader_time: 0.0146 (0.013357210159301757) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:57:15 -- STEP: 200/30063 -- GLOBAL_STEP: 200[0m |
|
| > loss_text_ce: 0.16569343209266663 (0.19600607939064507) |
|
| > loss_mel_ce: 5.862248420715332 (6.15806046485901) |
|
| > loss: 1.5069854259490967 (1.588516635298729) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2549 (0.3156870305538176) |
|
| > loader_time: 0.0104 (0.012890539169311523) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:57:41 -- STEP: 250/30063 -- GLOBAL_STEP: 250[0m |
|
| > loss_text_ce: 0.17059586942195892 (0.19009394341707234) |
|
| > loss_mel_ce: 6.081897735595703 (6.086897583007813) |
|
| > loss: 1.563123345375061 (1.5692478795051574) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2152 (0.3150361776351927) |
|
| > loader_time: 0.0088 (0.012582942008972168) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:58:06 -- STEP: 300/30063 -- GLOBAL_STEP: 300[0m |
|
| > loss_text_ce: 0.1637687385082245 (0.18456800366441412) |
|
| > loss_mel_ce: 5.864283561706543 (6.0412895059585585) |
|
| > loss: 1.5070130825042725 (1.5564643752574918) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1896 (0.31275563160578396) |
|
| > loader_time: 0.0087 (0.012389524777730304) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:58:33 -- STEP: 350/30063 -- GLOBAL_STEP: 350[0m |
|
| > loss_text_ce: 0.13504108786582947 (0.1791493505239487) |
|
| > loss_mel_ce: 5.591071128845215 (5.996060019901823) |
|
| > loss: 1.431528091430664 (1.5438023417336606) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.253 (0.3122274426051545) |
|
| > loader_time: 0.0093 (0.012196938650948659) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:58:58 -- STEP: 400/30063 -- GLOBAL_STEP: 400[0m |
|
| > loss_text_ce: 0.15164980292320251 (0.17355694336816677) |
|
| > loss_mel_ce: 5.829759120941162 (5.959608466625216) |
|
| > loss: 1.4953522682189941 (1.5332913517951972) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2888 (0.3111474370956418) |
|
| > loader_time: 0.0127 (0.012029411792755124) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:59:25 -- STEP: 450/30063 -- GLOBAL_STEP: 450[0m |
|
| > loss_text_ce: 0.10973162204027176 (0.16831256224049468) |
|
| > loss_mel_ce: 5.542464733123779 (5.920952761967979) |
|
| > loss: 1.4130491018295288 (1.5223163302739466) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3624 (0.3114083841111923) |
|
| > loader_time: 0.0126 (0.011909065246582029) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 07:59:50 -- STEP: 500/30063 -- GLOBAL_STEP: 500[0m |
|
| > loss_text_ce: 0.12962713837623596 (0.16308557140827185) |
|
| > loss_mel_ce: 5.6521897315979 (5.893536402702333) |
|
| > loss: 1.4454542398452759 (1.5141554930210117) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2397 (0.3100767297744749) |
|
| > loader_time: 0.0091 (0.011745912075042723) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:00:15 -- STEP: 550/30063 -- GLOBAL_STEP: 550[0m |
|
| > loss_text_ce: 0.10157814621925354 (0.15845132323828617) |
|
| > loss_mel_ce: 5.6893205642700195 (5.8659697064486425) |
|
| > loss: 1.44772469997406 (1.5061052573810925) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2883 (0.3090375276045364) |
|
| > loader_time: 0.0082 (0.011590709252790972) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:00:41 -- STEP: 600/30063 -- GLOBAL_STEP: 600[0m |
|
| > loss_text_ce: 0.0999237596988678 (0.15431842969109616) |
|
| > loss_mel_ce: 5.71621036529541 (5.843980432351433) |
|
| > loss: 1.4540334939956665 (1.4995747152964278) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1883 (0.308835740884145) |
|
| > loader_time: 0.0115 (0.011461192369461066) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:01:07 -- STEP: 650/30063 -- GLOBAL_STEP: 650[0m |
|
| > loss_text_ce: 0.09502509236335754 (0.15062232117240248) |
|
| > loss_mel_ce: 5.5468363761901855 (5.824419993620656) |
|
| > loss: 1.4104653596878052 (1.4937605780821586) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2258 (0.30844192541562604) |
|
| > loader_time: 0.0103 (0.011378423250638531) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:01:33 -- STEP: 700/30063 -- GLOBAL_STEP: 700[0m |
|
| > loss_text_ce: 0.10775484889745712 (0.1472807513816016) |
|
| > loss_mel_ce: 5.3456807136535645 (5.807394227981571) |
|
| > loss: 1.3633588552474976 (1.4886687440531594) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3332 (0.3085846209526063) |
|
| > loader_time: 0.0092 (0.011265302045004711) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:01:59 -- STEP: 750/30063 -- GLOBAL_STEP: 750[0m |
|
| > loss_text_ce: 0.08867921680212021 (0.14417442302902544) |
|
| > loss_mel_ce: 5.032188415527344 (5.788773053487144) |
|
| > loss: 1.280216932296753 (1.4832368680636088) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3739 (0.30869903278350835) |
|
| > loader_time: 0.0157 (0.011200013478597006) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:02:25 -- STEP: 800/30063 -- GLOBAL_STEP: 800[0m |
|
| > loss_text_ce: 0.10235428810119629 (0.1412876050826164) |
|
| > loss_mel_ce: 5.799641132354736 (5.769806320071223) |
|
| > loss: 1.475498914718628 (1.4777734805643559) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2885 (0.30898899078369135) |
|
| > loader_time: 0.0092 (0.011131909787654876) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:02:52 -- STEP: 850/30063 -- GLOBAL_STEP: 850[0m |
|
| > loss_text_ce: 0.09865696728229523 (0.13870997198364332) |
|
| > loss_mel_ce: 5.661565780639648 (5.752769415799311) |
|
| > loss: 1.4400557279586792 (1.4728698461196001) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3096 (0.30945486798005933) |
|
| > loader_time: 0.0083 (0.011041099043453441) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:03:18 -- STEP: 900/30063 -- GLOBAL_STEP: 900[0m |
|
| > loss_text_ce: 0.09370041638612747 (0.13638660865525415) |
|
| > loss_mel_ce: 5.630788803100586 (5.737670992745295) |
|
| > loss: 1.4311223030090332 (1.4685143992635938) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2504 (0.3097612884309556) |
|
| > loader_time: 0.009 (0.010977994600931804) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:03:44 -- STEP: 950/30063 -- GLOBAL_STEP: 950[0m |
|
| > loss_text_ce: 0.08709386736154556 (0.13417530608804598) |
|
| > loss_mel_ce: 5.793758869171143 (5.72142197608948) |
|
| > loss: 1.4702131748199463 (1.4638993195483558) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4417 (0.3091524294803016) |
|
| > loader_time: 0.0118 (0.010924491882324219) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:04:10 -- STEP: 1000/30063 -- GLOBAL_STEP: 1000[0m |
|
| > loss_text_ce: 0.08800945430994034 (0.1321571603417396) |
|
| > loss_mel_ce: 5.326340198516846 (5.705877014636995) |
|
| > loss: 1.3535873889923096 (1.4595085427761079) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1732 (0.30914154839515673) |
|
| > loader_time: 0.011 (0.010848899841308594) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:04:36 -- STEP: 1050/30063 -- GLOBAL_STEP: 1050[0m |
|
| > loss_text_ce: 0.08601665496826172 (0.13022883801942775) |
|
| > loss_mel_ce: 5.632408142089844 (5.693870799200877) |
|
| > loss: 1.4296061992645264 (1.4560249086788724) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2905 (0.30942515191577713) |
|
| > loader_time: 0.0091 (0.010805808703104654) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:05:02 -- STEP: 1100/30063 -- GLOBAL_STEP: 1100[0m |
|
| > loss_text_ce: 0.08602312207221985 (0.1284687467596747) |
|
| > loss_mel_ce: 5.536700248718262 (5.681885940378363) |
|
| > loss: 1.4056808948516846 (1.4525886709039857) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1894 (0.3091714057055385) |
|
| > loader_time: 0.0087 (0.010750555992126463) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:05:28 -- STEP: 1150/30063 -- GLOBAL_STEP: 1150[0m |
|
| > loss_text_ce: 0.08417614549398422 (0.12675709724426268) |
|
| > loss_mel_ce: 5.5014472007751465 (5.671958470966501) |
|
| > loss: 1.396405816078186 (1.4496788912234102) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2121 (0.30936030201289955) |
|
| > loader_time: 0.0093 (0.010705902472786282) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:05:53 -- STEP: 1200/30063 -- GLOBAL_STEP: 1200[0m |
|
| > loss_text_ce: 0.08322088420391083 (0.12516221993292367) |
|
| > loss_mel_ce: 5.500393390655518 (5.660428640445076) |
|
| > loss: 1.3959035873413086 (1.4463977145155273) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2519 (0.30891982714335103) |
|
| > loader_time: 0.0085 (0.010658098657925922) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:06:20 -- STEP: 1250/30063 -- GLOBAL_STEP: 1250[0m |
|
| > loss_text_ce: 0.08325894922018051 (0.12357091580629351) |
|
| > loss_mel_ce: 5.557662487030029 (5.648966109466552) |
|
| > loss: 1.4102303981781006 (1.443134255504609) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3241 (0.3090733589172363) |
|
| > loader_time: 0.0088 (0.010603929901123046) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:06:45 -- STEP: 1300/30063 -- GLOBAL_STEP: 1300[0m |
|
| > loss_text_ce: 0.09614972770214081 (0.12214811073472868) |
|
| > loss_mel_ce: 4.303592681884766 (5.636168299821704) |
|
| > loss: 1.0999356508255005 (1.4395791020760178) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2189 (0.30887425972865196) |
|
| > loader_time: 0.0091 (0.010556736542628361) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:07:11 -- STEP: 1350/30063 -- GLOBAL_STEP: 1350[0m |
|
| > loss_text_ce: 0.07779347896575928 (0.12072566411009543) |
|
| > loss_mel_ce: 5.503547668457031 (5.625556886814254) |
|
| > loss: 1.39533531665802 (1.4365706370494997) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4998 (0.30886782187002704) |
|
| > loader_time: 0.0104 (0.010519071155124246) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:07:36 -- STEP: 1400/30063 -- GLOBAL_STEP: 1400[0m |
|
| > loss_text_ce: 0.07625886052846909 (0.11942704156573332) |
|
| > loss_mel_ce: 5.332442760467529 (5.615913562093456) |
|
| > loss: 1.3521753549575806 (1.4338351504291813) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3527 (0.3083442379747119) |
|
| > loader_time: 0.0148 (0.010485467229570672) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:08:02 -- STEP: 1450/30063 -- GLOBAL_STEP: 1450[0m |
|
| > loss_text_ce: 0.08856235444545746 (0.11822888348123124) |
|
| > loss_mel_ce: 5.069084167480469 (5.604444723458121) |
|
| > loss: 1.2894116640090942 (1.4306684013070738) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3104 (0.30840879358094336) |
|
| > loader_time: 0.0091 (0.010447566098180322) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:08:28 -- STEP: 1500/30063 -- GLOBAL_STEP: 1500[0m |
|
| > loss_text_ce: 0.07408756762742996 (0.11703336800634863) |
|
| > loss_mel_ce: 5.485511302947998 (5.595548205375667) |
|
| > loss: 1.389899730682373 (1.4281453928947454) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.223 (0.3080576645533245) |
|
| > loader_time: 0.0086 (0.010408626079559334) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:08:54 -- STEP: 1550/30063 -- GLOBAL_STEP: 1550[0m |
|
| > loss_text_ce: 0.07401573657989502 (0.11589358337944555) |
|
| > loss_mel_ce: 5.453428268432617 (5.586482695917924) |
|
| > loss: 1.3818609714508057 (1.4255940691886415) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3012 (0.30811007007475855) |
|
| > loader_time: 0.0088 (0.010374061830582166) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:09:20 -- STEP: 1600/30063 -- GLOBAL_STEP: 1600[0m |
|
| > loss_text_ce: 0.07491308450698853 (0.1147708569839597) |
|
| > loss_mel_ce: 5.363227367401123 (5.577283202111716) |
|
| > loss: 1.3595350980758667 (1.423013514205814) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2884 (0.3082011656463145) |
|
| > loader_time: 0.0125 (0.010345256626605997) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:09:45 -- STEP: 1650/30063 -- GLOBAL_STEP: 1650[0m |
|
| > loss_text_ce: 0.08233626931905746 (0.11368184279311788) |
|
| > loss_mel_ce: 5.224329471588135 (5.568112325957323) |
|
| > loss: 1.3266664743423462 (1.4204485415689878) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3383 (0.30761829665212925) |
|
| > loader_time: 0.0099 (0.010329303596958979) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:10:11 -- STEP: 1700/30063 -- GLOBAL_STEP: 1700[0m |
|
| > loss_text_ce: 0.07145506888628006 (0.11268149183953512) |
|
| > loss_mel_ce: 5.418025970458984 (5.562068278929762) |
|
| > loss: 1.3723702430725098 (1.418687442050261) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2625 (0.30786982157651116) |
|
| > loader_time: 0.0089 (0.010314668907838718) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:10:36 -- STEP: 1750/30063 -- GLOBAL_STEP: 1750[0m |
|
| > loss_text_ce: 0.07609575986862183 (0.11171010547450612) |
|
| > loss_mel_ce: 5.219825267791748 (5.55399457114083) |
|
| > loss: 1.3239802122116089 (1.4164261685098927) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2758 (0.30738476508004325) |
|
| > loader_time: 0.0111 (0.010299347741263261) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:11:02 -- STEP: 1800/30063 -- GLOBAL_STEP: 1800[0m |
|
| > loss_text_ce: 0.07980663329362869 (0.11075186309715114) |
|
| > loss_mel_ce: 5.36020565032959 (5.548088555600904) |
|
| > loss: 1.360003113746643 (1.4147101041343482) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1887 (0.30729703691270616) |
|
| > loader_time: 0.0085 (0.010279223124186205) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:11:27 -- STEP: 1850/30063 -- GLOBAL_STEP: 1850[0m |
|
| > loss_text_ce: 0.0747850239276886 (0.10978481787684803) |
|
| > loss_mel_ce: 4.964337348937988 (5.540190805744476) |
|
| > loss: 1.2597806453704834 (1.4124939053123067) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2834 (0.30689654724017995) |
|
| > loader_time: 0.0086 (0.010249082719957513) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:11:54 -- STEP: 1900/30063 -- GLOBAL_STEP: 1900[0m |
|
| > loss_text_ce: 0.07936915755271912 (0.10889650933248433) |
|
| > loss_mel_ce: 4.668835639953613 (5.5313092786387354) |
|
| > loss: 1.1870511770248413 (1.4100514463374494) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.253 (0.30733379602432254) |
|
| > loader_time: 0.0085 (0.010230237308301431) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:12:19 -- STEP: 1950/30063 -- GLOBAL_STEP: 1950[0m |
|
| > loss_text_ce: 0.0797397643327713 (0.1079766525137119) |
|
| > loss_mel_ce: 5.299657344818115 (5.525013010073927) |
|
| > loss: 1.3448492288589478 (1.4082474148579138) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2582 (0.3070032523228572) |
|
| > loader_time: 0.009 (0.01021249575492664) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:12:43 -- STEP: 2000/30063 -- GLOBAL_STEP: 2000[0m |
|
| > loss_text_ce: 0.0669279620051384 (0.10712874733656647) |
|
| > loss_mel_ce: 5.1178178787231445 (5.517483374118801) |
|
| > loss: 1.2961864471435547 (1.4061530295610432) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2542 (0.3064018021821976) |
|
| > loader_time: 0.0087 (0.010187365174293524) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_2000.pth |
|
|
|
[1m --> TIME: 2024-10-26 08:13:21 -- STEP: 2050/30063 -- GLOBAL_STEP: 2050[0m |
|
| > loss_text_ce: 0.06811168044805527 (0.10628551141881362) |
|
| > loss_mel_ce: 4.559199810028076 (5.510345500853) |
|
| > loss: 1.1568279266357422 (1.4041577524092144) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2848 (0.30655269320418194) |
|
| > loader_time: 0.0088 (0.010183970986343018) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:13:47 -- STEP: 2100/30063 -- GLOBAL_STEP: 2100[0m |
|
| > loss_text_ce: 0.07062087208032608 (0.10543713966295834) |
|
| > loss_mel_ce: 5.500833511352539 (5.503551035381494) |
|
| > loss: 1.3928636312484741 (1.4022470431100758) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2453 (0.30659473873320076) |
|
| > loader_time: 0.009 (0.010166156746092303) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:14:13 -- STEP: 2150/30063 -- GLOBAL_STEP: 2150[0m |
|
| > loss_text_ce: 0.06449185311794281 (0.10463515500862933) |
|
| > loss_mel_ce: 5.3209991455078125 (5.4979623597167215) |
|
| > loss: 1.3463727235794067 (1.4006493780224818) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2544 (0.30670245226039444) |
|
| > loader_time: 0.0085 (0.010146812061930818) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:14:40 -- STEP: 2200/30063 -- GLOBAL_STEP: 2200[0m |
|
| > loss_text_ce: 0.07365085184574127 (0.10385430124842311) |
|
| > loss_mel_ce: 5.484344005584717 (5.4911244722929835) |
|
| > loss: 1.3894987106323242 (1.3987446926127773) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2052 (0.3070101110501721) |
|
| > loader_time: 0.0084 (0.010129004066640687) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:15:06 -- STEP: 2250/30063 -- GLOBAL_STEP: 2250[0m |
|
| > loss_text_ce: 0.07574379444122314 (0.10312586959534226) |
|
| > loss_mel_ce: 5.031594276428223 (5.484766653378802) |
|
| > loss: 1.276834487915039 (1.3969731299877162) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3132 (0.30696369330088286) |
|
| > loader_time: 0.0089 (0.010111840777926985) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:15:32 -- STEP: 2300/30063 -- GLOBAL_STEP: 2300[0m |
|
| > loss_text_ce: 0.07440293580293655 (0.10237585138529544) |
|
| > loss_mel_ce: 4.590554714202881 (5.478043002563973) |
|
| > loss: 1.1662393808364868 (1.395104712532914) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1525 (0.30696037862611825) |
|
| > loader_time: 0.0119 (0.010098232600999924) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:15:57 -- STEP: 2350/30063 -- GLOBAL_STEP: 2350[0m |
|
| > loss_text_ce: 0.06387393921613693 (0.10166311325545013) |
|
| > loss_mel_ce: 5.345189571380615 (5.472762038149732) |
|
| > loss: 1.3522658348083496 (1.39360628688589) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3617 (0.30689309039014423) |
|
| > loader_time: 0.0092 (0.010082799627425828) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:16:23 -- STEP: 2400/30063 -- GLOBAL_STEP: 2400[0m |
|
| > loss_text_ce: 0.06882283836603165 (0.10095946935005495) |
|
| > loss_mel_ce: 5.448531150817871 (5.4681817411383005) |
|
| > loss: 1.3793385028839111 (1.3922853016604977) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2752 (0.30689351518948865) |
|
| > loader_time: 0.0092 (0.01006083836158117) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:16:50 -- STEP: 2450/30063 -- GLOBAL_STEP: 2450[0m |
|
| > loss_text_ce: 0.06075205281376839 (0.1002809236578796) |
|
| > loss_mel_ce: 5.319052219390869 (5.461250479270002) |
|
| > loss: 1.3449510335922241 (1.3903828499755075) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2589 (0.30705850912600147) |
|
| > loader_time: 0.0089 (0.010044105685487095) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:17:14 -- STEP: 2500/30063 -- GLOBAL_STEP: 2500[0m |
|
| > loss_text_ce: 0.06374214589595795 (0.09958283366113915) |
|
| > loss_mel_ce: 4.876726150512695 (5.454756479358678) |
|
| > loss: 1.2351170778274536 (1.388584827423095) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2705 (0.30664564971923836) |
|
| > loader_time: 0.0093 (0.010029112148284926) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:17:40 -- STEP: 2550/30063 -- GLOBAL_STEP: 2550[0m |
|
| > loss_text_ce: 0.05650924891233444 (0.09893472672239254) |
|
| > loss_mel_ce: 4.584807395935059 (5.448961325907244) |
|
| > loss: 1.1603291034698486 (1.3869740123374776) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2553 (0.306775053996666) |
|
| > loader_time: 0.0085 (0.010015119571311812) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:18:07 -- STEP: 2600/30063 -- GLOBAL_STEP: 2600[0m |
|
| > loss_text_ce: 0.061880555003881454 (0.09828463365108939) |
|
| > loss_mel_ce: 5.471291542053223 (5.443524964864442) |
|
| > loss: 1.3832930326461792 (1.3854523987036467) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3232 (0.3070918783774744) |
|
| > loader_time: 0.0098 (0.010002823242774387) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:18:34 -- STEP: 2650/30063 -- GLOBAL_STEP: 2650[0m |
|
| > loss_text_ce: 0.06976098567247391 (0.09766374249925046) |
|
| > loss_mel_ce: 5.448820114135742 (5.437433556700658) |
|
| > loss: 1.3796452283859253 (1.383774323913284) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4661 (0.3071978550137215) |
|
| > loader_time: 0.0091 (0.009987944566978614) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:19:00 -- STEP: 2700/30063 -- GLOBAL_STEP: 2700[0m |
|
| > loss_text_ce: 0.05855318531394005 (0.09703293374153216) |
|
| > loss_mel_ce: 4.8042755126953125 (5.431825043095487) |
|
| > loss: 1.2157071828842163 (1.3822144935307652) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2083 (0.307398458145283) |
|
| > loader_time: 0.0094 (0.009979187117682578) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:19:25 -- STEP: 2750/30063 -- GLOBAL_STEP: 2750[0m |
|
| > loss_text_ce: 0.06550798565149307 (0.09644117969816397) |
|
| > loss_mel_ce: 4.832806587219238 (5.4272411524165785) |
|
| > loss: 1.224578619003296 (1.3809205824245072) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2149 (0.30720100134069267) |
|
| > loader_time: 0.01 (0.00997125833684749) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:19:49 -- STEP: 2800/30063 -- GLOBAL_STEP: 2800[0m |
|
| > loss_text_ce: 0.05796299874782562 (0.09583733841377726) |
|
| > loss_mel_ce: 5.177628517150879 (5.42183058338506) |
|
| > loss: 1.308897852897644 (1.3794169797641855) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2318 (0.3068307880844391) |
|
| > loader_time: 0.0087 (0.009960115977696018) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:20:15 -- STEP: 2850/30063 -- GLOBAL_STEP: 2850[0m |
|
| > loss_text_ce: 0.07211673259735107 (0.09525114514195099) |
|
| > loss_mel_ce: 4.86887788772583 (5.41721613423866) |
|
| > loss: 1.2352486848831177 (1.37811681923113) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2765 (0.30675500585321813) |
|
| > loader_time: 0.0084 (0.00994930735805579) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:20:40 -- STEP: 2900/30063 -- GLOBAL_STEP: 2900[0m |
|
| > loss_text_ce: 0.06425466388463974 (0.09470726624516593) |
|
| > loss_mel_ce: 5.489013671875 (5.411897129765875) |
|
| > loss: 1.3883171081542969 (1.3766510983582163) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2555 (0.30660548086824113) |
|
| > loader_time: 0.0101 (0.009935188951163463) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:21:05 -- STEP: 2950/30063 -- GLOBAL_STEP: 2950[0m |
|
| > loss_text_ce: 0.06667879223823547 (0.0941485169353123) |
|
| > loss_mel_ce: 4.914728164672852 (5.40638057328887) |
|
| > loss: 1.245351791381836 (1.3751322717989867) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2765 (0.3064562492855527) |
|
| > loader_time: 0.0091 (0.009921366885557023) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:21:32 -- STEP: 3000/30063 -- GLOBAL_STEP: 3000[0m |
|
| > loss_text_ce: 0.05804649740457535 (0.09361584443226471) |
|
| > loss_mel_ce: 5.580506801605225 (5.402270357211437) |
|
| > loss: 1.4096382856369019 (1.3739715496698972) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2514 (0.30669864741961217) |
|
| > loader_time: 0.0091 (0.009911420663197858) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:21:58 -- STEP: 3050/30063 -- GLOBAL_STEP: 3050[0m |
|
| > loss_text_ce: 0.05688001587986946 (0.09307807677837683) |
|
| > loss_mel_ce: 5.228001594543457 (5.39718115830031) |
|
| > loss: 1.3212203979492188 (1.3725648081888877) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.284 (0.30661608742885976) |
|
| > loader_time: 0.009 (0.00989944919210968) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:22:24 -- STEP: 3100/30063 -- GLOBAL_STEP: 3100[0m |
|
| > loss_text_ce: 0.0635627955198288 (0.09255758854650696) |
|
| > loss_mel_ce: 5.0295538902282715 (5.391927004245025) |
|
| > loss: 1.2732791900634766 (1.371121147717195) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2254 (0.3066209843850914) |
|
| > loader_time: 0.0085 (0.009887698235050344) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:22:50 -- STEP: 3150/30063 -- GLOBAL_STEP: 3150[0m |
|
| > loss_text_ce: 0.05579125136137009 (0.09203267107289004) |
|
| > loss_mel_ce: 4.962165355682373 (5.38733748125652) |
|
| > loss: 1.2544891834259033 (1.369842537622599) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3112 (0.3066844329379864) |
|
| > loader_time: 0.0091 (0.009876208229670468) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:23:17 -- STEP: 3200/30063 -- GLOBAL_STEP: 3200[0m |
|
| > loss_text_ce: 0.062379587441682816 (0.09152061841683476) |
|
| > loss_mel_ce: 5.582221031188965 (5.382664303705101) |
|
| > loss: 1.4111500978469849 (1.36854623008519) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2542 (0.30686685062944974) |
|
| > loader_time: 0.0092 (0.009864901453256629) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:23:42 -- STEP: 3250/30063 -- GLOBAL_STEP: 3250[0m |
|
| > loss_text_ce: 0.056953128427267075 (0.09102743740035951) |
|
| > loss_mel_ce: 5.310308933258057 (5.377411424563486) |
|
| > loss: 1.34181547164917 (1.36710971494821) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2888 (0.30684112130678687) |
|
| > loader_time: 0.0085 (0.00985392827254077) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:24:08 -- STEP: 3300/30063 -- GLOBAL_STEP: 3300[0m |
|
| > loss_text_ce: 0.0671331137418747 (0.09057583946847568) |
|
| > loss_mel_ce: 5.217059135437012 (5.372881090641026) |
|
| > loss: 1.3210480213165283 (1.3658642319476928) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3195 (0.30684599435690746) |
|
| > loader_time: 0.0104 (0.009841780951528855) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:24:35 -- STEP: 3350/30063 -- GLOBAL_STEP: 3350[0m |
|
| > loss_text_ce: 0.05577133968472481 (0.09010711561721665) |
|
| > loss_mel_ce: 4.900613784790039 (5.368441508350092) |
|
| > loss: 1.2390962839126587 (1.3646371554616634) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3985 (0.30712972420365126) |
|
| > loader_time: 0.0086 (0.009833562480869595) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:25:01 -- STEP: 3400/30063 -- GLOBAL_STEP: 3400[0m |
|
| > loss_text_ce: 0.06137121841311455 (0.0896515914600563) |
|
| > loss_mel_ce: 5.19257116317749 (5.364051356105247) |
|
| > loss: 1.3134856224060059 (1.3634257363221187) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2472 (0.3071863407247213) |
|
| > loader_time: 0.0087 (0.009822374091428885) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:25:27 -- STEP: 3450/30063 -- GLOBAL_STEP: 3450[0m |
|
| > loss_text_ce: 0.059456028044223785 (0.08919331718182232) |
|
| > loss_mel_ce: 4.8889312744140625 (5.360089647251631) |
|
| > loss: 1.2370967864990234 (1.3623207403956943) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3239 (0.3072151103227041) |
|
| > loader_time: 0.0092 (0.009815978658372094) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:25:54 -- STEP: 3500/30063 -- GLOBAL_STEP: 3500[0m |
|
| > loss_text_ce: 0.059247225522994995 (0.08876542833447469) |
|
| > loss_mel_ce: 4.851404190063477 (5.355922462122785) |
|
| > loss: 1.2276628017425537 (1.3611719719682385) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2737 (0.3073270673070641) |
|
| > loader_time: 0.009 (0.009805948938642246) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:26:20 -- STEP: 3550/30063 -- GLOBAL_STEP: 3550[0m |
|
| > loss_text_ce: 0.06208595633506775 (0.08831732398922187) |
|
| > loss_mel_ce: 5.375740051269531 (5.3517854431985175) |
|
| > loss: 1.3594565391540527 (1.3600256911129982) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3805 (0.30733932448105133) |
|
| > loader_time: 0.0098 (0.009795999459817392) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:26:47 -- STEP: 3600/30063 -- GLOBAL_STEP: 3600[0m |
|
| > loss_text_ce: 0.06425635516643524 (0.08790372783628615) |
|
| > loss_mel_ce: 4.582530975341797 (5.347199749350552) |
|
| > loss: 1.1616967916488647 (1.35877586851517) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3591 (0.3075332593917852) |
|
| > loader_time: 0.0103 (0.00979210257530214) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:27:13 -- STEP: 3650/30063 -- GLOBAL_STEP: 3650[0m |
|
| > loss_text_ce: 0.05682969465851784 (0.08749756801822427) |
|
| > loss_mel_ce: 5.275410175323486 (5.3434848795198455) |
|
| > loss: 1.3330600261688232 (1.3577456111777284) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3366 (0.30768314054567575) |
|
| > loader_time: 0.0087 (0.009783586279986671) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:27:39 -- STEP: 3700/30063 -- GLOBAL_STEP: 3700[0m |
|
| > loss_text_ce: 0.05106568709015846 (0.08708993616539092) |
|
| > loss_mel_ce: 4.455092430114746 (5.3391103113664204) |
|
| > loss: 1.1265395879745483 (1.3565500611227876) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2017 (0.30768884987444545) |
|
| > loader_time: 0.0091 (0.009777096542152214) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:28:05 -- STEP: 3750/30063 -- GLOBAL_STEP: 3750[0m |
|
| > loss_text_ce: 0.06302288919687271 (0.08669127986232451) |
|
| > loss_mel_ce: 5.117846488952637 (5.335145044390364) |
|
| > loss: 1.2952173948287964 (1.3554590803782112) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4454 (0.30763209616343234) |
|
| > loader_time: 0.011 (0.009769464810689306) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:28:31 -- STEP: 3800/30063 -- GLOBAL_STEP: 3800[0m |
|
| > loss_text_ce: 0.05548869073390961 (0.08628803236998238) |
|
| > loss_mel_ce: 5.010197162628174 (5.330449713343072) |
|
| > loss: 1.2664214372634888 (1.3541844356687411) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3444 (0.30778440425270537) |
|
| > loader_time: 0.0101 (0.009762681973607931) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:28:57 -- STEP: 3850/30063 -- GLOBAL_STEP: 3850[0m |
|
| > loss_text_ce: 0.05964277312159538 (0.0859230433181897) |
|
| > loss_mel_ce: 4.641628265380859 (5.325238827854012) |
|
| > loss: 1.1753177642822266 (1.3527904669959792) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3151 (0.30771857373126194) |
|
| > loader_time: 0.0087 (0.00975645052922238) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:29:23 -- STEP: 3900/30063 -- GLOBAL_STEP: 3900[0m |
|
| > loss_text_ce: 0.06471210718154907 (0.08553808374760254) |
|
| > loss_mel_ce: 5.375728607177734 (5.321331409001965) |
|
| > loss: 1.3601101636886597 (1.3517173723991067) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3196 (0.30783938994774507) |
|
| > loader_time: 0.0094 (0.00975006311367721) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:29:49 -- STEP: 3950/30063 -- GLOBAL_STEP: 3950[0m |
|
| > loss_text_ce: 0.060686301440000534 (0.0851611256401374) |
|
| > loss_mel_ce: 4.59771728515625 (5.317355869691587) |
|
| > loss: 1.1646008491516113 (1.3506292479551258) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3386 (0.3079400326330456) |
|
| > loader_time: 0.0091 (0.009744126042233252) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:30:16 -- STEP: 4000/30063 -- GLOBAL_STEP: 4000[0m |
|
| > loss_text_ce: 0.05586684122681618 (0.08479565447755169) |
|
| > loss_mel_ce: 4.8931989669799805 (5.314127012073997) |
|
| > loss: 1.2372664213180542 (1.3497306658029524) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2088 (0.30810717231035284) |
|
| > loader_time: 0.0083 (0.009735280275344862) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_4000.pth |
|
|
|
[1m --> TIME: 2024-10-26 08:30:56 -- STEP: 4050/30063 -- GLOBAL_STEP: 4050[0m |
|
| > loss_text_ce: 0.052190251648426056 (0.08444626804303251) |
|
| > loss_mel_ce: 4.670748710632324 (5.309610804334103) |
|
| > loss: 1.1807347536087036 (1.3485142673533608) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2956 (0.30827877904162043) |
|
| > loader_time: 0.0083 (0.009738614235395281) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:31:22 -- STEP: 4100/30063 -- GLOBAL_STEP: 4100[0m |
|
| > loss_text_ce: 0.05852193757891655 (0.0841007058595013) |
|
| > loss_mel_ce: 5.107022285461426 (5.305386142032906) |
|
| > loss: 1.2913860082626343 (1.3473717111930583) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2485 (0.30832245239397427) |
|
| > loader_time: 0.0085 (0.00973010848208173) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:31:48 -- STEP: 4150/30063 -- GLOBAL_STEP: 4150[0m |
|
| > loss_text_ce: 0.05039399117231369 (0.08375424219901315) |
|
| > loss_mel_ce: 4.78805685043335 (5.301750245956057) |
|
| > loss: 1.2096127271652222 (1.3463761213170446) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3087 (0.3082484912872319) |
|
| > loader_time: 0.0088 (0.009720529016241984) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:32:14 -- STEP: 4200/30063 -- GLOBAL_STEP: 4200[0m |
|
| > loss_text_ce: 0.05939150229096413 (0.0834163748171359) |
|
| > loss_mel_ce: 4.867889881134033 (5.298356269200647) |
|
| > loss: 1.2318203449249268 (1.345443160292645) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2443 (0.3082044482231145) |
|
| > loader_time: 0.0086 (0.00971100239526659) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:32:40 -- STEP: 4250/30063 -- GLOBAL_STEP: 4250[0m |
|
| > loss_text_ce: 0.053591351956129074 (0.0830794850412538) |
|
| > loss_mel_ce: 4.552379131317139 (5.294268745310171) |
|
| > loss: 1.1514925956726074 (1.3443370568471764) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2516 (0.30825748174330814) |
|
| > loader_time: 0.0091 (0.009706943231470458) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:33:06 -- STEP: 4300/30063 -- GLOBAL_STEP: 4300[0m |
|
| > loss_text_ce: 0.061251912266016006 (0.08275208839756816) |
|
| > loss_mel_ce: 4.968867301940918 (5.290114767606872) |
|
| > loss: 1.257529854774475 (1.3432167131817583) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2238 (0.3082856611872833) |
|
| > loader_time: 0.0085 (0.009699547623479101) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:33:33 -- STEP: 4350/30063 -- GLOBAL_STEP: 4350[0m |
|
| > loss_text_ce: 0.05414481461048126 (0.08243993345262682) |
|
| > loss_mel_ce: 5.35730504989624 (5.28703852357536) |
|
| > loss: 1.3528624773025513 (1.3423696133871155) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3017 (0.3084913237889613) |
|
| > loader_time: 0.0091 (0.009692860142938036) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:33:59 -- STEP: 4400/30063 -- GLOBAL_STEP: 4400[0m |
|
| > loss_text_ce: 0.05556824058294296 (0.08212703285230837) |
|
| > loss_mel_ce: 5.031979084014893 (5.283835186633201) |
|
| > loss: 1.2718868255615234 (1.341490553956136) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2063 (0.3085539044033404) |
|
| > loader_time: 0.0088 (0.009686627875674867) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:34:25 -- STEP: 4450/30063 -- GLOBAL_STEP: 4450[0m |
|
| > loss_text_ce: 0.052425116300582886 (0.08182114905520776) |
|
| > loss_mel_ce: 5.110321998596191 (5.279547021469374) |
|
| > loss: 1.2906867265701294 (1.3403420417362344) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4415 (0.30855855175618424) |
|
| > loader_time: 0.009 (0.009681190640738854) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:34:51 -- STEP: 4500/30063 -- GLOBAL_STEP: 4500[0m |
|
| > loss_text_ce: 0.05320815369486809 (0.08151754578948027) |
|
| > loss_mel_ce: 5.224514961242676 (5.276120165877871) |
|
| > loss: 1.3194308280944824 (1.3394094270202805) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2894 (0.30852230554156895) |
|
| > loader_time: 0.0092 (0.009674134254455577) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:35:16 -- STEP: 4550/30063 -- GLOBAL_STEP: 4550[0m |
|
| > loss_text_ce: 0.05023062974214554 (0.0812040988519625) |
|
| > loss_mel_ce: 5.157011032104492 (5.272658821514675) |
|
| > loss: 1.301810383796692 (1.3384657292182585) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2756 (0.30835745062146935) |
|
| > loader_time: 0.0091 (0.00966801947289772) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:35:41 -- STEP: 4600/30063 -- GLOBAL_STEP: 4600[0m |
|
| > loss_text_ce: 0.053258538246154785 (0.08090741043262516) |
|
| > loss_mel_ce: 4.84642219543457 (5.26945495237475) |
|
| > loss: 1.2249201536178589 (1.3375905898472498) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2202 (0.30824133598286263) |
|
| > loader_time: 0.0091 (0.009662288116372158) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:36:07 -- STEP: 4650/30063 -- GLOBAL_STEP: 4650[0m |
|
| > loss_text_ce: 0.05655081942677498 (0.0806219706476056) |
|
| > loss_mel_ce: 5.497552394866943 (5.265216525344438) |
|
| > loss: 1.3885258436203003 (1.3364596231650252) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3771 (0.3082388992207032) |
|
| > loader_time: 0.0089 (0.009654397605567862) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:36:32 -- STEP: 4700/30063 -- GLOBAL_STEP: 4700[0m |
|
| > loss_text_ce: 0.05432265251874924 (0.08034848345879557) |
|
| > loss_mel_ce: 4.778601169586182 (5.261200698639489) |
|
| > loss: 1.208230972290039 (1.3353872947363112) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2468 (0.30811896993758897) |
|
| > loader_time: 0.0094 (0.009647638036849666) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:36:57 -- STEP: 4750/30063 -- GLOBAL_STEP: 4750[0m |
|
| > loss_text_ce: 0.054096613079309464 (0.0800752571813371) |
|
| > loss_mel_ce: 5.384282112121582 (5.258289113245517) |
|
| > loss: 1.359594702720642 (1.3345910918712582) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2872 (0.3080228351291863) |
|
| > loader_time: 0.0083 (0.009640800827427931) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:37:22 -- STEP: 4800/30063 -- GLOBAL_STEP: 4800[0m |
|
| > loss_text_ce: 0.061632998287677765 (0.079796956415133) |
|
| > loss_mel_ce: 5.053612232208252 (5.25527614369989) |
|
| > loss: 1.2788113355636597 (1.333768274200456) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1771 (0.30790709629654955) |
|
| > loader_time: 0.0114 (0.009634631226460159) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:37:47 -- STEP: 4850/30063 -- GLOBAL_STEP: 4850[0m |
|
| > loss_text_ce: 0.05280528962612152 (0.07951196558082228) |
|
| > loss_mel_ce: 4.944235801696777 (5.251706095282568) |
|
| > loss: 1.2492603063583374 (1.3328045143909035) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5044 (0.30781903458624843) |
|
| > loader_time: 0.0123 (0.009628454680295358) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:38:13 -- STEP: 4900/30063 -- GLOBAL_STEP: 4900[0m |
|
| > loss_text_ce: 0.05001556873321533 (0.07923891535142852) |
|
| > loss_mel_ce: 5.33284330368042 (5.248878259804786) |
|
| > loss: 1.3457146883010864 (1.3320292929848816) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3974 (0.307817061288017) |
|
| > loader_time: 0.0103 (0.00962495181025293) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:38:38 -- STEP: 4950/30063 -- GLOBAL_STEP: 4950[0m |
|
| > loss_text_ce: 0.057402871549129486 (0.07898112643622987) |
|
| > loss_mel_ce: 4.846681594848633 (5.246179242615752) |
|
| > loss: 1.22602117061615 (1.33129009149291) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4024 (0.3077281044950395) |
|
| > loader_time: 0.0088 (0.009619294754182472) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:39:04 -- STEP: 5000/30063 -- GLOBAL_STEP: 5000[0m |
|
| > loss_text_ce: 0.04803409054875374 (0.0787204534493386) |
|
| > loss_mel_ce: 5.035665988922119 (5.242769097089774) |
|
| > loss: 1.2709250450134277 (1.3303723868727653) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2026 (0.30763668522834847) |
|
| > loader_time: 0.0087 (0.00961424646377566) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:39:30 -- STEP: 5050/30063 -- GLOBAL_STEP: 5050[0m |
|
| > loss_text_ce: 0.049694254994392395 (0.07846560239496794) |
|
| > loss_mel_ce: 4.9324750900268555 (5.239026062040048) |
|
| > loss: 1.245542287826538 (1.3293729153836094) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3366 (0.307683503462537) |
|
| > loader_time: 0.0086 (0.009608216332917195) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:39:56 -- STEP: 5100/30063 -- GLOBAL_STEP: 5100[0m |
|
| > loss_text_ce: 0.050695694983005524 (0.07821530453057261) |
|
| > loss_mel_ce: 4.998305797576904 (5.235262287691527) |
|
| > loss: 1.2622504234313965 (1.3283693974275197) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3483 (0.307639473372815) |
|
| > loader_time: 0.0094 (0.009605188556745962) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:40:22 -- STEP: 5150/30063 -- GLOBAL_STEP: 5150[0m |
|
| > loss_text_ce: 0.049359772354364395 (0.07797325653430905) |
|
| > loss_mel_ce: 4.994658946990967 (5.232206886856298) |
|
| > loss: 1.2610046863555908 (1.3275450352904847) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3781 (0.3076929405360551) |
|
| > loader_time: 0.0098 (0.009603027788180764) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:40:48 -- STEP: 5200/30063 -- GLOBAL_STEP: 5200[0m |
|
| > loss_text_ce: 0.05708217993378639 (0.0777273942974324) |
|
| > loss_mel_ce: 4.99580192565918 (5.228980866441358) |
|
| > loss: 1.263221025466919 (1.326677064609066) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3029 (0.30775526069677833) |
|
| > loader_time: 0.0088 (0.009598385599943315) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:41:14 -- STEP: 5250/30063 -- GLOBAL_STEP: 5250[0m |
|
| > loss_text_ce: 0.05572541058063507 (0.07748530706834218) |
|
| > loss_mel_ce: 4.9542059898376465 (5.225410131590704) |
|
| > loss: 1.2524828910827637 (1.3257238590944354) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3069 (0.30780510725293886) |
|
| > loader_time: 0.0087 (0.009593558084397096) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:41:40 -- STEP: 5300/30063 -- GLOBAL_STEP: 5300[0m |
|
| > loss_text_ce: 0.05386144667863846 (0.07725407427422827) |
|
| > loss_mel_ce: 4.7178850173950195 (5.22191834346303) |
|
| > loss: 1.192936658859253 (1.3247931038995935) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1883 (0.3077082051421115) |
|
| > loader_time: 0.0083 (0.009588770416547675) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:42:06 -- STEP: 5350/30063 -- GLOBAL_STEP: 5350[0m |
|
| > loss_text_ce: 0.050288598984479904 (0.07702725212150624) |
|
| > loss_mel_ce: 5.080891132354736 (5.21858347264405) |
|
| > loss: 1.2827949523925781 (1.323902680751318) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3596 (0.30782888916051304) |
|
| > loader_time: 0.0089 (0.009583431136942355) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:42:32 -- STEP: 5400/30063 -- GLOBAL_STEP: 5400[0m |
|
| > loss_text_ce: 0.05106084793806076 (0.07679917003476489) |
|
| > loss_mel_ce: 4.948300838470459 (5.215010965400264) |
|
| > loss: 1.2498403787612915 (1.322952533485711) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2516 (0.3078046407964498) |
|
| > loader_time: 0.0102 (0.009578969522758768) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:42:59 -- STEP: 5450/30063 -- GLOBAL_STEP: 5450[0m |
|
| > loss_text_ce: 0.05196305736899376 (0.07658719975740515) |
|
| > loss_mel_ce: 4.577043533325195 (5.212189451576368) |
|
| > loss: 1.1572515964508057 (1.322194162454078) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2998 (0.3079055452784272) |
|
| > loader_time: 0.0087 (0.00957394188697185) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:43:24 -- STEP: 5500/30063 -- GLOBAL_STEP: 5500[0m |
|
| > loss_text_ce: 0.04803841933608055 (0.07636471346223872) |
|
| > loss_mel_ce: 5.292352676391602 (5.208950527321206) |
|
| > loss: 1.3350977897644043 (1.3213288097923428) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2291 (0.30783557198264433) |
|
| > loader_time: 0.0103 (0.009568363666534418) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:43:51 -- STEP: 5550/30063 -- GLOBAL_STEP: 5550[0m |
|
| > loss_text_ce: 0.05387811362743378 (0.07613851040131848) |
|
| > loss_mel_ce: 4.544703483581543 (5.206089579049529) |
|
| > loss: 1.149645447731018 (1.320557021998067) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2587 (0.3079100893209651) |
|
| > loader_time: 0.0085 (0.009562606210107199) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:44:16 -- STEP: 5600/30063 -- GLOBAL_STEP: 5600[0m |
|
| > loss_text_ce: 0.050927676260471344 (0.07591794898666947) |
|
| > loss_mel_ce: 4.936334609985352 (5.203281921616617) |
|
| > loss: 1.24681556224823 (1.3197999671420846) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2297 (0.30788117843014906) |
|
| > loader_time: 0.0089 (0.009557633655411853) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:44:42 -- STEP: 5650/30063 -- GLOBAL_STEP: 5650[0m |
|
| > loss_text_ce: 0.049536194652318954 (0.07570459768110122) |
|
| > loss_mel_ce: 4.739019870758057 (5.200053286214838) |
|
| > loss: 1.197139024734497 (1.3189394704957942) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2896 (0.3078766363278962) |
|
| > loader_time: 0.0085 (0.009550985699206321) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:45:07 -- STEP: 5700/30063 -- GLOBAL_STEP: 5700[0m |
|
| > loss_text_ce: 0.04668520390987396 (0.07548934124279424) |
|
| > loss_mel_ce: 5.0926408767700195 (5.196563154061627) |
|
| > loss: 1.2848315238952637 (1.3180131233470447) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2395 (0.30773977877801006) |
|
| > loader_time: 0.0091 (0.009546013380351808) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:45:32 -- STEP: 5750/30063 -- GLOBAL_STEP: 5750[0m |
|
| > loss_text_ce: 0.0504402332007885 (0.07528607162703622) |
|
| > loss_mel_ce: 4.741614818572998 (5.193402985489879) |
|
| > loss: 1.1980137825012207 (1.317172263736306) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3093 (0.30766080051919775) |
|
| > loader_time: 0.0083 (0.009539858486341384) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:45:57 -- STEP: 5800/30063 -- GLOBAL_STEP: 5800[0m |
|
| > loss_text_ce: 0.05087360739707947 (0.07507364639883908) |
|
| > loss_mel_ce: 4.553732872009277 (5.190338049387101) |
|
| > loss: 1.1511516571044922 (1.3163529234199642) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2069 (0.3076071765916111) |
|
| > loader_time: 0.0087 (0.00953650865061529) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:46:22 -- STEP: 5850/30063 -- GLOBAL_STEP: 5850[0m |
|
| > loss_text_ce: 0.04882040247321129 (0.07487271299219522) |
|
| > loss_mel_ce: 5.060157775878906 (5.187742671436724) |
|
| > loss: 1.2772445678710938 (1.3156538455812288) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2981 (0.30755144706139315) |
|
| > loader_time: 0.0113 (0.009532623331770932) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:46:49 -- STEP: 5900/30063 -- GLOBAL_STEP: 5900[0m |
|
| > loss_text_ce: 0.048869259655475616 (0.0746715444730498) |
|
| > loss_mel_ce: 4.846346378326416 (5.184533368894604) |
|
| > loss: 1.2238038778305054 (1.3148012278544638) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3048 (0.30758367595026187) |
|
| > loader_time: 0.0089 (0.009527626967026005) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:47:14 -- STEP: 5950/30063 -- GLOBAL_STEP: 5950[0m |
|
| > loss_text_ce: 0.047826722264289856 (0.07446985409602887) |
|
| > loss_mel_ce: 4.583195686340332 (5.180775097157772) |
|
| > loss: 1.1577556133270264 (1.313811237341209) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.231 (0.3075581868356028) |
|
| > loader_time: 0.0086 (0.009524337183527583) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:47:39 -- STEP: 6000/30063 -- GLOBAL_STEP: 6000[0m |
|
| > loss_text_ce: 0.04876580461859703 (0.0742642337394256) |
|
| > loss_mel_ce: 4.826101303100586 (5.177349878668782) |
|
| > loss: 1.2187167406082153 (1.3129035276075134) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1881 (0.3074538544813807) |
|
| > loader_time: 0.0084 (0.00952050487200419) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_6000.pth |
|
|
|
[1m --> TIME: 2024-10-26 08:48:19 -- STEP: 6050/30063 -- GLOBAL_STEP: 6050[0m |
|
| > loss_text_ce: 0.054143842309713364 (0.07407664288235588) |
|
| > loss_mel_ce: 4.892397403717041 (5.174122432953061) |
|
| > loss: 1.2366353273391724 (1.3120497684340762) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2324 (0.30756384439705) |
|
| > loader_time: 0.0083 (0.00951935362224737) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:48:46 -- STEP: 6100/30063 -- GLOBAL_STEP: 6100[0m |
|
| > loss_text_ce: 0.045703545212745667 (0.07389048568904391) |
|
| > loss_mel_ce: 4.344536781311035 (5.170498295729271) |
|
| > loss: 1.0975600481033325 (1.3110971948260097) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.23 (0.30757187655714857) |
|
| > loader_time: 0.0087 (0.00951429894713105) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:49:12 -- STEP: 6150/30063 -- GLOBAL_STEP: 6150[0m |
|
| > loss_text_ce: 0.04807350039482117 (0.07370351681076896) |
|
| > loss_mel_ce: 4.908357620239258 (5.167895246017263) |
|
| > loss: 1.2391077280044556 (1.310399690188037) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.401 (0.3076262278285471) |
|
| > loader_time: 0.0087 (0.009509249392563736) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:49:36 -- STEP: 6200/30063 -- GLOBAL_STEP: 6200[0m |
|
| > loss_text_ce: 0.04511073976755142 (0.0735146028890965) |
|
| > loss_mel_ce: 4.413634300231934 (5.1647894039077045) |
|
| > loss: 1.1146862506866455 (1.30957600121536) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2195 (0.30744003030561784) |
|
| > loader_time: 0.0082 (0.009503971107544439) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:50:02 -- STEP: 6250/30063 -- GLOBAL_STEP: 6250[0m |
|
| > loss_text_ce: 0.05372001975774765 (0.07332887484490866) |
|
| > loss_mel_ce: 5.218628883361816 (5.161942618141165) |
|
| > loss: 1.3180872201919556 (1.3088178728008226) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3007 (0.3074469510269182) |
|
| > loader_time: 0.0082 (0.009500380287170413) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:50:26 -- STEP: 6300/30063 -- GLOBAL_STEP: 6300[0m |
|
| > loss_text_ce: 0.050848305225372314 (0.07314446925229964) |
|
| > loss_mel_ce: 4.92686128616333 (5.158804636039424) |
|
| > loss: 1.2444274425506592 (1.3079872758899374) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2759 (0.30729720603852195) |
|
| > loader_time: 0.0085 (0.009494357298290913) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:50:52 -- STEP: 6350/30063 -- GLOBAL_STEP: 6350[0m |
|
| > loss_text_ce: 0.05048312246799469 (0.07296217626767357) |
|
| > loss_mel_ce: 4.990358352661133 (5.1554244580982225) |
|
| > loss: 1.260210394859314 (1.3070966581284487) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.337 (0.30731659449930415) |
|
| > loader_time: 0.0086 (0.00949081413389191) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:51:20 -- STEP: 6400/30063 -- GLOBAL_STEP: 6400[0m |
|
| > loss_text_ce: 0.05153748020529747 (0.0727860816090832) |
|
| > loss_mel_ce: 5.4646196365356445 (5.1535943691804995) |
|
| > loss: 1.3790392875671387 (1.3065951122529762) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3188 (0.3074952596798555) |
|
| > loader_time: 0.0093 (0.009487704224884512) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:51:46 -- STEP: 6450/30063 -- GLOBAL_STEP: 6450[0m |
|
| > loss_text_ce: 0.04921098053455353 (0.0726089353946058) |
|
| > loss_mel_ce: 4.615845203399658 (5.150817711297854) |
|
| > loss: 1.1662640571594238 (1.3058566612605857) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2673 (0.3075618502520791) |
|
| > loader_time: 0.0091 (0.009482960516168162) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:52:12 -- STEP: 6500/30063 -- GLOBAL_STEP: 6500[0m |
|
| > loss_text_ce: 0.05639159679412842 (0.07243927258539651) |
|
| > loss_mel_ce: 4.94861364364624 (5.148278458118438) |
|
| > loss: 1.2512513399124146 (1.3051794322270571) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3584 (0.30757776249372215) |
|
| > loader_time: 0.0098 (0.00947986932901235) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:52:38 -- STEP: 6550/30063 -- GLOBAL_STEP: 6550[0m |
|
| > loss_text_ce: 0.051986660808324814 (0.07226459981262218) |
|
| > loss_mel_ce: 4.255697250366211 (5.1454107075247135) |
|
| > loss: 1.076920986175537 (1.3044188263944045) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2862 (0.30757568388494977) |
|
| > loader_time: 0.0086 (0.009478177041497844) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:53:04 -- STEP: 6600/30063 -- GLOBAL_STEP: 6600[0m |
|
| > loss_text_ce: 0.04714253917336464 (0.07208799334242931) |
|
| > loss_mel_ce: 4.200692653656006 (5.142138416478127) |
|
| > loss: 1.0619587898254395 (1.303556602001186) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2395 (0.30760203650503604) |
|
| > loader_time: 0.0092 (0.00947571837540828) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:53:32 -- STEP: 6650/30063 -- GLOBAL_STEP: 6650[0m |
|
| > loss_text_ce: 0.05410097911953926 (0.07192627017995468) |
|
| > loss_mel_ce: 5.115808486938477 (5.139731081482162) |
|
| > loss: 1.2924773693084717 (1.302914337556161) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2998 (0.3077805868665093) |
|
| > loader_time: 0.0085 (0.009473313245558196) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:53:59 -- STEP: 6700/30063 -- GLOBAL_STEP: 6700[0m |
|
| > loss_text_ce: 0.04336101934313774 (0.07175930735391017) |
|
| > loss_mel_ce: 5.180060863494873 (5.1371050313693365) |
|
| > loss: 1.3058555126190186 (1.3022160842881234) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4085 (0.3078166410460415) |
|
| > loader_time: 0.0099 (0.009471960743861406) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:54:24 -- STEP: 6750/30063 -- GLOBAL_STEP: 6750[0m |
|
| > loss_text_ce: 0.047823090106248856 (0.07159254969104564) |
|
| > loss_mel_ce: 4.645129680633545 (5.1344579917766415) |
|
| > loss: 1.1732381582260132 (1.3015126349749349) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2598 (0.30775234702781534) |
|
| > loader_time: 0.0091 (0.009469611591762957) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:54:49 -- STEP: 6800/30063 -- GLOBAL_STEP: 6800[0m |
|
| > loss_text_ce: 0.04295102506875992 (0.0714244175959816) |
|
| > loss_mel_ce: 5.38251256942749 (5.131899861307704) |
|
| > loss: 1.3563659191131592 (1.3008310693765348) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2092 (0.30767295865451727) |
|
| > loader_time: 0.0097 (0.009467599181567914) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:55:14 -- STEP: 6850/30063 -- GLOBAL_STEP: 6850[0m |
|
| > loss_text_ce: 0.040934547781944275 (0.07126103512072648) |
|
| > loss_mel_ce: 4.77911901473999 (5.129008587816335) |
|
| > loss: 1.205013394355774 (1.3000674053613246) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2337 (0.30758801564683147) |
|
| > loader_time: 0.0085 (0.009463490430456) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:55:40 -- STEP: 6900/30063 -- GLOBAL_STEP: 6900[0m |
|
| > loss_text_ce: 0.05115421488881111 (0.07110199623511751) |
|
| > loss_mel_ce: 4.8123369216918945 (5.12636384487152) |
|
| > loss: 1.2158727645874023 (1.2993664598724077) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2231 (0.30756727101146447) |
|
| > loader_time: 0.0089 (0.00945973040401071) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:56:05 -- STEP: 6950/30063 -- GLOBAL_STEP: 6950[0m |
|
| > loss_text_ce: 0.05364755541086197 (0.07094418296282241) |
|
| > loss_mel_ce: 4.720203876495361 (5.123611506180796) |
|
| > loss: 1.19346284866333 (1.2986389218988996) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.29 (0.3075364296727915) |
|
| > loader_time: 0.009 (0.009459517053562952) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:56:31 -- STEP: 7000/30063 -- GLOBAL_STEP: 7000[0m |
|
| > loss_text_ce: 0.05403774976730347 (0.07079904444249618) |
|
| > loss_mel_ce: 4.834413051605225 (5.120805518422807) |
|
| > loss: 1.2221126556396484 (1.297901140366278) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3045 (0.3074961162294674) |
|
| > loader_time: 0.0088 (0.009458149433135978) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:56:57 -- STEP: 7050/30063 -- GLOBAL_STEP: 7050[0m |
|
| > loss_text_ce: 0.05605943873524666 (0.07064579639451728) |
|
| > loss_mel_ce: 4.523179531097412 (5.11837789062067) |
|
| > loss: 1.1448097229003906 (1.2972559213807358) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4236 (0.30749494637158037) |
|
| > loader_time: 0.009 (0.009454369308255234) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:57:25 -- STEP: 7100/30063 -- GLOBAL_STEP: 7100[0m |
|
| > loss_text_ce: 0.04986662417650223 (0.07050368365687387) |
|
| > loss_mel_ce: 4.875361442565918 (5.11584436873315) |
|
| > loss: 1.231307029724121 (1.2965870127207757) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2078 (0.307694830659411) |
|
| > loader_time: 0.0084 (0.0094521239441885) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:57:50 -- STEP: 7150/30063 -- GLOBAL_STEP: 7150[0m |
|
| > loss_text_ce: 0.05578547716140747 (0.07034925732683468) |
|
| > loss_mel_ce: 4.955790996551514 (5.113094672256415) |
|
| > loss: 1.2528941631317139 (1.295860982041255) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3415 (0.3076202752039996) |
|
| > loader_time: 0.0091 (0.009448732162688987) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:58:15 -- STEP: 7200/30063 -- GLOBAL_STEP: 7200[0m |
|
| > loss_text_ce: 0.05076083540916443 (0.07019972331252784) |
|
| > loss_mel_ce: 4.482229232788086 (5.110276120073265) |
|
| > loss: 1.1332474946975708 (1.2951189605229392) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2769 (0.30749342474672664) |
|
| > loader_time: 0.009 (0.009445868697431345) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:58:39 -- STEP: 7250/30063 -- GLOBAL_STEP: 7250[0m |
|
| > loss_text_ce: 0.0449727438390255 (0.07005203231449791) |
|
| > loss_mel_ce: 4.635336875915527 (5.107826707346685) |
|
| > loss: 1.1700774431228638 (1.2944696846501542) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2087 (0.3073897930342589) |
|
| > loader_time: 0.0096 (0.009442852086034306) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:59:05 -- STEP: 7300/30063 -- GLOBAL_STEP: 7300[0m |
|
| > loss_text_ce: 0.048395540565252304 (0.06990216300554883) |
|
| > loss_mel_ce: 5.156826019287109 (5.104894975113541) |
|
| > loss: 1.3013054132461548 (1.2936992842530515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1731 (0.307364421544011) |
|
| > loader_time: 0.0088 (0.00944006825146609) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:59:31 -- STEP: 7350/30063 -- GLOBAL_STEP: 7350[0m |
|
| > loss_text_ce: 0.044843968003988266 (0.06975436385036732) |
|
| > loss_mel_ce: 4.734724521636963 (5.102093723777199) |
|
| > loss: 1.194892168045044 (1.2929620216006288) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3598 (0.3073976058051713) |
|
| > loader_time: 0.0086 (0.009438275934076626) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 08:59:57 -- STEP: 7400/30063 -- GLOBAL_STEP: 7400[0m |
|
| > loss_text_ce: 0.04968007281422615 (0.0696153411556136) |
|
| > loss_mel_ce: 5.202208042144775 (5.099391696968593) |
|
| > loss: 1.312972068786621 (1.292251759227865) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2706 (0.30741635164699166) |
|
| > loader_time: 0.009 (0.00943746038385339) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:00:23 -- STEP: 7450/30063 -- GLOBAL_STEP: 7450[0m |
|
| > loss_text_ce: 0.048379987478256226 (0.06948040632993582) |
|
| > loss_mel_ce: 4.924644470214844 (5.096992545351885) |
|
| > loss: 1.2432560920715332 (1.2916182375514231) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3054 (0.3074529529417934) |
|
| > loader_time: 0.0098 (0.009436228099285349) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:00:50 -- STEP: 7500/30063 -- GLOBAL_STEP: 7500[0m |
|
| > loss_text_ce: 0.04645639285445213 (0.06934010148098071) |
|
| > loss_mel_ce: 4.962783336639404 (5.094743250052133) |
|
| > loss: 1.2523099184036255 (1.291020837537444) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3205 (0.3074977883338941) |
|
| > loader_time: 0.0086 (0.009433635552724196) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:01:15 -- STEP: 7550/30063 -- GLOBAL_STEP: 7550[0m |
|
| > loss_text_ce: 0.04705824330449104 (0.06920398574347134) |
|
| > loss_mel_ce: 5.064231872558594 (5.092304064422253) |
|
| > loss: 1.277822494506836 (1.2903770122701723) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3818 (0.3074381890202213) |
|
| > loader_time: 0.01 (0.009430843283798514) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:01:40 -- STEP: 7600/30063 -- GLOBAL_STEP: 7600[0m |
|
| > loss_text_ce: 0.04761073365807533 (0.06906843905288138) |
|
| > loss_mel_ce: 5.081933975219727 (5.089699823542644) |
|
| > loss: 1.2823861837387085 (1.2896920653942348) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2875 (0.3073494929702671) |
|
| > loader_time: 0.0089 (0.009430322490240391) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:02:07 -- STEP: 7650/30063 -- GLOBAL_STEP: 7650[0m |
|
| > loss_text_ce: 0.04644805192947388 (0.06892944757065742) |
|
| > loss_mel_ce: 5.051183700561523 (5.08739608789581) |
|
| > loss: 1.274407982826233 (1.2890813836474788) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3589 (0.3074791598475849) |
|
| > loader_time: 0.0088 (0.009428170802546475) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:02:33 -- STEP: 7700/30063 -- GLOBAL_STEP: 7700[0m |
|
| > loss_text_ce: 0.0478401780128479 (0.06879798726892317) |
|
| > loss_mel_ce: 4.146048069000244 (5.084770686812214) |
|
| > loss: 1.0484720468521118 (1.288392168283459) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2478 (0.30750714106993365) |
|
| > loader_time: 0.0089 (0.00942642502970509) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:02:59 -- STEP: 7750/30063 -- GLOBAL_STEP: 7750[0m |
|
| > loss_text_ce: 0.04729926958680153 (0.06866483971092009) |
|
| > loss_mel_ce: 4.452090263366699 (5.081776839994614) |
|
| > loss: 1.124847412109375 (1.2876104197348281) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2975 (0.30749182821089344) |
|
| > loader_time: 0.0099 (0.009424184737666954) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:03:25 -- STEP: 7800/30063 -- GLOBAL_STEP: 7800[0m |
|
| > loss_text_ce: 0.04890133813023567 (0.06852941507187027) |
|
| > loss_mel_ce: 4.190849304199219 (5.079268223658585) |
|
| > loss: 1.0599377155303955 (1.286949409452765) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2059 (0.3075223493270397) |
|
| > loader_time: 0.0086 (0.009422722504689136) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:03:52 -- STEP: 7850/30063 -- GLOBAL_STEP: 7850[0m |
|
| > loss_text_ce: 0.04709719866514206 (0.06839888308220989) |
|
| > loss_mel_ce: 4.656945705413818 (5.076435230370539) |
|
| > loss: 1.1760107278823853 (1.2862085281663604) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1628 (0.3075831049718687) |
|
| > loader_time: 0.0148 (0.009422303157247548) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:04:17 -- STEP: 7900/30063 -- GLOBAL_STEP: 7900[0m |
|
| > loss_text_ce: 0.05796726420521736 (0.06827097406775891) |
|
| > loss_mel_ce: 4.585546016693115 (5.0731271419042265) |
|
| > loss: 1.160878300666809 (1.2853495288121515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1843 (0.3075533524344251) |
|
| > loader_time: 0.0084 (0.009420771719534178) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:04:43 -- STEP: 7950/30063 -- GLOBAL_STEP: 7950[0m |
|
| > loss_text_ce: 0.04201190546154976 (0.06814106286977822) |
|
| > loss_mel_ce: 4.645221710205078 (5.070994603798823) |
|
| > loss: 1.1718083620071411 (1.2847839164958776) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3145 (0.3075668084546467) |
|
| > loader_time: 0.0095 (0.00941868362186839) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:05:11 -- STEP: 8000/30063 -- GLOBAL_STEP: 8000[0m |
|
| > loss_text_ce: 0.05196619778871536 (0.06801372760813683) |
|
| > loss_mel_ce: 4.982473373413086 (5.068517632007598) |
|
| > loss: 1.2586098909378052 (1.2841328397467697) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3584 (0.3077086949050438) |
|
| > loader_time: 0.0096 (0.009416900187730783) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_8000.pth |
|
|
|
[1m --> TIME: 2024-10-26 09:05:50 -- STEP: 8050/30063 -- GLOBAL_STEP: 8050[0m |
|
| > loss_text_ce: 0.04565136879682541 (0.06789201162385274) |
|
| > loss_mel_ce: 4.208895206451416 (5.066229526211756) |
|
| > loss: 1.0636366605758667 (1.2835303842873274) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3615 (0.3077111552516878) |
|
| > loader_time: 0.0091 (0.009418802972165687) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:06:17 -- STEP: 8100/30063 -- GLOBAL_STEP: 8100[0m |
|
| > loss_text_ce: 0.04153524711728096 (0.06776188364183461) |
|
| > loss_mel_ce: 4.566102027893066 (5.064151602262331) |
|
| > loss: 1.151909351348877 (1.2829783713008116) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2512 (0.3077945953239641) |
|
| > loader_time: 0.0091 (0.009417468206382085) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:06:42 -- STEP: 8150/30063 -- GLOBAL_STEP: 8150[0m |
|
| > loss_text_ce: 0.044901292771101 (0.06763758468573079) |
|
| > loss_mel_ce: 5.0430474281311035 (5.061811980908633) |
|
| > loss: 1.2719871997833252 (1.282362391180784) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3351 (0.3077500936297563) |
|
| > loader_time: 0.0084 (0.009414776936630523) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:07:09 -- STEP: 8200/30063 -- GLOBAL_STEP: 8200[0m |
|
| > loss_text_ce: 0.0454082153737545 (0.06751905771790118) |
|
| > loss_mel_ce: 4.519289493560791 (5.059019703138165) |
|
| > loss: 1.1411744356155396 (1.2816346900056015) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2869 (0.3078093310972551) |
|
| > loader_time: 0.0092 (0.009412290468448539) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:07:34 -- STEP: 8250/30063 -- GLOBAL_STEP: 8250[0m |
|
| > loss_text_ce: 0.04671340435743332 (0.06739923784407702) |
|
| > loss_mel_ce: 4.8445916175842285 (5.056781493013554) |
|
| > loss: 1.2228262424468994 (1.2810451825026277) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2844 (0.30781348384510504) |
|
| > loader_time: 0.0082 (0.009410045768275399) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:08:00 -- STEP: 8300/30063 -- GLOBAL_STEP: 8300[0m |
|
| > loss_text_ce: 0.05163051187992096 (0.06727993517665258) |
|
| > loss_mel_ce: 4.690332412719727 (5.054585831624914) |
|
| > loss: 1.1854907274246216 (1.2804664414546547) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2079 (0.3078222240310129) |
|
| > loader_time: 0.0085 (0.009408139177115563) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:08:26 -- STEP: 8350/30063 -- GLOBAL_STEP: 8350[0m |
|
| > loss_text_ce: 0.04417100176215172 (0.06715430419869765) |
|
| > loss_mel_ce: 4.941776275634766 (5.051836240334425) |
|
| > loss: 1.246486783027649 (1.279747635904184) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2839 (0.30775566469409615) |
|
| > loader_time: 0.0095 (0.009407010306855158) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:08:50 -- STEP: 8400/30063 -- GLOBAL_STEP: 8400[0m |
|
| > loss_text_ce: 0.04773438721895218 (0.06704179907306317) |
|
| > loss_mel_ce: 5.108583450317383 (5.049579876718067) |
|
| > loss: 1.2890794277191162 (1.2791554187663936) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2894 (0.307648855050406) |
|
| > loader_time: 0.0089 (0.009405551779837828) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:09:15 -- STEP: 8450/30063 -- GLOBAL_STEP: 8450[0m |
|
| > loss_text_ce: 0.04428543895483017 (0.06692515764319332) |
|
| > loss_mel_ce: 4.722553730010986 (5.047003762256464) |
|
| > loss: 1.1917097568511963 (1.2784822298013219) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2553 (0.3075593126455013) |
|
| > loader_time: 0.0084 (0.009403263266975352) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:09:40 -- STEP: 8500/30063 -- GLOBAL_STEP: 8500[0m |
|
| > loss_text_ce: 0.04003987833857536 (0.06680960359906447) |
|
| > loss_mel_ce: 4.371427059173584 (5.045015277357662) |
|
| > loss: 1.102866768836975 (1.2779562201149297) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2101 (0.30753717430900224) |
|
| > loader_time: 0.0097 (0.009401686976937676) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:10:06 -- STEP: 8550/30063 -- GLOBAL_STEP: 8550[0m |
|
| > loss_text_ce: 0.05339425802230835 (0.0667010460144769) |
|
| > loss_mel_ce: 4.546338081359863 (5.042802318104522) |
|
| > loss: 1.149933099746704 (1.2773758409385765) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4699 (0.3075226592459883) |
|
| > loader_time: 0.0096 (0.00939881589677598) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:10:33 -- STEP: 8600/30063 -- GLOBAL_STEP: 8600[0m |
|
| > loss_text_ce: 0.04726944491267204 (0.06658767819794457) |
|
| > loss_mel_ce: 4.55790376663208 (5.040967856950533) |
|
| > loss: 1.1512932777404785 (1.276888883675251) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2092 (0.3075814642185399) |
|
| > loader_time: 0.0087 (0.009396943075712328) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:10:59 -- STEP: 8650/30063 -- GLOBAL_STEP: 8650[0m |
|
| > loss_text_ce: 0.04324669763445854 (0.06647360552209995) |
|
| > loss_mel_ce: 4.4818572998046875 (5.038614866885141) |
|
| > loss: 1.13127601146698 (1.276272117981329) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3103 (0.30763903915537677) |
|
| > loader_time: 0.0087 (0.00939516323839308) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:11:25 -- STEP: 8700/30063 -- GLOBAL_STEP: 8700[0m |
|
| > loss_text_ce: 0.051467884331941605 (0.0663593505421418) |
|
| > loss_mel_ce: 4.216485977172852 (5.0364380571211855) |
|
| > loss: 1.066988468170166 (1.2756993517930455) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2445 (0.30762604461319243) |
|
| > loader_time: 0.0085 (0.009393067195497703) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:11:50 -- STEP: 8750/30063 -- GLOBAL_STEP: 8750[0m |
|
| > loss_text_ce: 0.0551118329167366 (0.066253345688326) |
|
| > loss_mel_ce: 4.701315402984619 (5.034400744301929) |
|
| > loss: 1.189106822013855 (1.275163522311616) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3043 (0.30755533133915597) |
|
| > loader_time: 0.0091 (0.009391172845023013) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:12:16 -- STEP: 8800/30063 -- GLOBAL_STEP: 8800[0m |
|
| > loss_text_ce: 0.04350483790040016 (0.06614231976976784) |
|
| > loss_mel_ce: 4.2583112716674805 (5.031916452917182) |
|
| > loss: 1.0754539966583252 (1.2745146929540383) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2035 (0.30755023468624576) |
|
| > loader_time: 0.0086 (0.00938868793574246) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:12:42 -- STEP: 8850/30063 -- GLOBAL_STEP: 8850[0m |
|
| > loss_text_ce: 0.05048595368862152 (0.06603203036859207) |
|
| > loss_mel_ce: 4.905280113220215 (5.029709189517347) |
|
| > loss: 1.2389415502548218 (1.2739353047925832) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2311 (0.3075820515519492) |
|
| > loader_time: 0.0082 (0.009386972923063282) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:13:10 -- STEP: 8900/30063 -- GLOBAL_STEP: 8900[0m |
|
| > loss_text_ce: 0.04182222858071327 (0.06592351649375104) |
|
| > loss_mel_ce: 4.195362567901611 (5.027495626653174) |
|
| > loss: 1.0592962503433228 (1.2733547856298681) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2057 (0.3077082822296067) |
|
| > loader_time: 0.0084 (0.009386230934871707) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:13:35 -- STEP: 8950/30063 -- GLOBAL_STEP: 8950[0m |
|
| > loss_text_ce: 0.04534178972244263 (0.06581567967445819) |
|
| > loss_mel_ce: 3.9669606685638428 (5.025229556840218) |
|
| > loss: 1.0030755996704102 (1.2727613089737235) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2552 (0.30766561116586283) |
|
| > loader_time: 0.009 (0.009384270273773351) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:14:01 -- STEP: 9000/30063 -- GLOBAL_STEP: 9000[0m |
|
| > loss_text_ce: 0.04479904845356941 (0.06570983089423836) |
|
| > loss_mel_ce: 4.215223789215088 (5.0228632046381545) |
|
| > loss: 1.0650056600570679 (1.2721432587173203) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1871 (0.30765716698434703) |
|
| > loader_time: 0.0097 (0.009382897933324166) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:14:25 -- STEP: 9050/30063 -- GLOBAL_STEP: 9050[0m |
|
| > loss_text_ce: 0.039955902844667435 (0.06559978377588538) |
|
| > loss_mel_ce: 4.754630088806152 (5.020990268354249) |
|
| > loss: 1.1986465454101562 (1.2716475128666422) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2596 (0.30754815778679717) |
|
| > loader_time: 0.0096 (0.009380926005748082) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:14:50 -- STEP: 9100/30063 -- GLOBAL_STEP: 9100[0m |
|
| > loss_text_ce: 0.04646781459450722 (0.06549145685652116) |
|
| > loss_mel_ce: 4.84979772567749 (5.019107038555556) |
|
| > loss: 1.2240663766860962 (1.2711496236625563) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2246 (0.3074864152761617) |
|
| > loader_time: 0.0094 (0.00937947870610833) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:15:16 -- STEP: 9150/30063 -- GLOBAL_STEP: 9150[0m |
|
| > loss_text_ce: 0.048718325793743134 (0.06539056486648605) |
|
| > loss_mel_ce: 4.25131368637085 (5.017017595937336) |
|
| > loss: 1.0750080347061157 (1.2706020400628317) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2336 (0.30749383577232264) |
|
| > loader_time: 0.0085 (0.009377983854116628) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:15:43 -- STEP: 9200/30063 -- GLOBAL_STEP: 9200[0m |
|
| > loss_text_ce: 0.0471118688583374 (0.0652840959475091) |
|
| > loss_mel_ce: 4.605229377746582 (5.014790821593734) |
|
| > loss: 1.1630853414535522 (1.270018729275978) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2076 (0.307537568341132) |
|
| > loader_time: 0.0091 (0.009376272999722012) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:16:09 -- STEP: 9250/30063 -- GLOBAL_STEP: 9250[0m |
|
| > loss_text_ce: 0.04292181506752968 (0.06518247394827563) |
|
| > loss_mel_ce: 4.478394508361816 (5.012488087705659) |
|
| > loss: 1.1303291320800781 (1.26941764032518) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.262 (0.3075567583908919) |
|
| > loader_time: 0.0085 (0.009373795251588551) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:16:35 -- STEP: 9300/30063 -- GLOBAL_STEP: 9300[0m |
|
| > loss_text_ce: 0.04348050802946091 (0.0650859934131624) |
|
| > loss_mel_ce: 4.4285078048706055 (5.01027032567608) |
|
| > loss: 1.1179970502853394 (1.2688390796671578) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3043 (0.3075873072685749) |
|
| > loader_time: 0.0095 (0.00937042564474125) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:16:59 -- STEP: 9350/30063 -- GLOBAL_STEP: 9350[0m |
|
| > loss_text_ce: 0.04959333688020706 (0.06498527519165821) |
|
| > loss_mel_ce: 4.9205451011657715 (5.008190027048234) |
|
| > loss: 1.2425346374511719 (1.2682938254580727) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4285 (0.3075054761050235) |
|
| > loader_time: 0.0087 (0.009367147154986533) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:17:25 -- STEP: 9400/30063 -- GLOBAL_STEP: 9400[0m |
|
| > loss_text_ce: 0.04399513453245163 (0.06488285649726352) |
|
| > loss_mel_ce: 4.437255382537842 (5.006124274172675) |
|
| > loss: 1.1203125715255737 (1.267751782549183) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2483 (0.3074770924639211) |
|
| > loader_time: 0.0085 (0.009363563010033129) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:17:51 -- STEP: 9450/30063 -- GLOBAL_STEP: 9450[0m |
|
| > loss_text_ce: 0.045519113540649414 (0.06478508448119817) |
|
| > loss_mel_ce: 4.412610054016113 (5.004127570182552) |
|
| > loss: 1.114532232284546 (1.2672281635879792) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.335 (0.3075090731767136) |
|
| > loader_time: 0.0099 (0.009359977863453038) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:18:17 -- STEP: 9500/30063 -- GLOBAL_STEP: 9500[0m |
|
| > loss_text_ce: 0.05057214945554733 (0.06468588528429196) |
|
| > loss_mel_ce: 4.834811210632324 (5.0018192673482345) |
|
| > loss: 1.2213457822799683 (1.2666262880751917) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2501 (0.3075194097318165) |
|
| > loader_time: 0.0077 (0.009355699012154014) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:18:44 -- STEP: 9550/30063 -- GLOBAL_STEP: 9550[0m |
|
| > loss_text_ce: 0.04791411757469177 (0.06458384960694766) |
|
| > loss_mel_ce: 4.45664644241333 (4.999583621299702) |
|
| > loss: 1.1261401176452637 (1.2660418676331369) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3974 (0.30755159370562346) |
|
| > loader_time: 0.0082 (0.009351899137047563) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:19:10 -- STEP: 9600/30063 -- GLOBAL_STEP: 9600[0m |
|
| > loss_text_ce: 0.046952612698078156 (0.06448913765411504) |
|
| > loss_mel_ce: 4.986007213592529 (4.997377066314218) |
|
| > loss: 1.258239984512329 (1.265466550911462) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2717 (0.3076145727932469) |
|
| > loader_time: 0.0083 (0.009349546507000917) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:19:37 -- STEP: 9650/30063 -- GLOBAL_STEP: 9650[0m |
|
| > loss_text_ce: 0.04784470424056053 (0.06439239426559422) |
|
| > loss_mel_ce: 4.7191996574401855 (4.994989513288506) |
|
| > loss: 1.1917611360549927 (1.2648454768237627) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4927 (0.3076365560571163) |
|
| > loader_time: 0.0089 (0.009346292945387446) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:20:02 -- STEP: 9700/30063 -- GLOBAL_STEP: 9700[0m |
|
| > loss_text_ce: 0.0444919653236866 (0.06429525005525534) |
|
| > loss_mel_ce: 4.810189723968506 (4.992949280837148) |
|
| > loss: 1.2136703729629517 (1.2643111326460947) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2055 (0.3075870000210023) |
|
| > loader_time: 0.0082 (0.009342599455843262) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:20:27 -- STEP: 9750/30063 -- GLOBAL_STEP: 9750[0m |
|
| > loss_text_ce: 0.05829209089279175 (0.06420120623860595) |
|
| > loss_mel_ce: 5.063032627105713 (4.990763189633688) |
|
| > loss: 1.2803311347961426 (1.2637410988991047) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3879 (0.30758349741422386) |
|
| > loader_time: 0.0093 (0.009340144573113848) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:20:53 -- STEP: 9800/30063 -- GLOBAL_STEP: 9800[0m |
|
| > loss_text_ce: 0.04380590841174126 (0.06410940858592473) |
|
| > loss_mel_ce: 4.697413444519043 (4.98880662020372) |
|
| > loss: 1.185304880142212 (1.2632290071674726) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2079 (0.30755612587442366) |
|
| > loader_time: 0.0085 (0.009337990892176715) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:21:19 -- STEP: 9850/30063 -- GLOBAL_STEP: 9850[0m |
|
| > loss_text_ce: 0.05170084908604622 (0.06402088557856929) |
|
| > loss_mel_ce: 4.406652450561523 (4.986744991220194) |
|
| > loss: 1.1145883798599243 (1.2626914691501443) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2504 (0.30754950961485883) |
|
| > loader_time: 0.0085 (0.009333648173337039) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:21:45 -- STEP: 9900/30063 -- GLOBAL_STEP: 9900[0m |
|
| > loss_text_ce: 0.04140520840883255 (0.06392819652954722) |
|
| > loss_mel_ce: 4.494514465332031 (4.984458376759229) |
|
| > loss: 1.1339799165725708 (1.2620966432672527) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2764 (0.3075814427751499) |
|
| > loader_time: 0.0086 (0.009331044741351184) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:22:11 -- STEP: 9950/30063 -- GLOBAL_STEP: 9950[0m |
|
| > loss_text_ce: 0.04541175439953804 (0.06383677862359938) |
|
| > loss_mel_ce: 4.4480977058410645 (4.982165972671318) |
|
| > loss: 1.1233773231506348 (1.2615006877429484) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.261 (0.3075829853604203) |
|
| > loader_time: 0.0085 (0.009327885829024567) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:22:37 -- STEP: 10000/30063 -- GLOBAL_STEP: 10000[0m |
|
| > loss_text_ce: 0.04455357417464256 (0.06374829961173221) |
|
| > loss_mel_ce: 4.299884796142578 (4.980085641765593) |
|
| > loss: 1.0861096382141113 (1.2609584852993456) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2243 (0.3076040564298643) |
|
| > loader_time: 0.0088 (0.009324054002761826) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_10000.pth |
|
|
|
[1m --> TIME: 2024-10-26 09:23:14 -- STEP: 10050/30063 -- GLOBAL_STEP: 10050[0m |
|
| > loss_text_ce: 0.04175692796707153 (0.0636585009242615) |
|
| > loss_mel_ce: 4.36986780166626 (4.978244080211395) |
|
| > loss: 1.1029062271118164 (1.260475645272883) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3089 (0.3075430592257004) |
|
| > loader_time: 0.0087 (0.009324142256779445) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:23:39 -- STEP: 10100/30063 -- GLOBAL_STEP: 10100[0m |
|
| > loss_text_ce: 0.045051198452711105 (0.06356994441719623) |
|
| > loss_mel_ce: 4.352637767791748 (4.97634028354494) |
|
| > loss: 1.0994222164154053 (1.2599775569745775) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2283 (0.3075051268728665) |
|
| > loader_time: 0.0086 (0.009319894124965849) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:24:05 -- STEP: 10150/30063 -- GLOBAL_STEP: 10150[0m |
|
| > loss_text_ce: 0.04328029602766037 (0.06348312948321166) |
|
| > loss_mel_ce: 4.290757656097412 (4.974175616297235) |
|
| > loss: 1.0835094451904297 (1.2594146864754785) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2499 (0.3075034415428287) |
|
| > loader_time: 0.0086 (0.009316884966319411) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:24:32 -- STEP: 10200/30063 -- GLOBAL_STEP: 10200[0m |
|
| > loss_text_ce: 0.04577929526567459 (0.06339595576857804) |
|
| > loss_mel_ce: 4.149557113647461 (4.972113497116996) |
|
| > loss: 1.0488340854644775 (1.2588773632517019) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2402 (0.3075460172634511) |
|
| > loader_time: 0.0095 (0.009313935836156197) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:24:57 -- STEP: 10250/30063 -- GLOBAL_STEP: 10250[0m |
|
| > loss_text_ce: 0.056496694684028625 (0.06330684762720643) |
|
| > loss_mel_ce: 4.128530979156494 (4.9700258190806395) |
|
| > loss: 1.0462568998336792 (1.2583331667446473) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2594 (0.3075111797146692) |
|
| > loader_time: 0.0081 (0.009311463797964683) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:25:22 -- STEP: 10300/30063 -- GLOBAL_STEP: 10300[0m |
|
| > loss_text_ce: 0.04111476242542267 (0.06322176657219235) |
|
| > loss_mel_ce: 4.747211933135986 (4.967864628879775) |
|
| > loss: 1.1970816850662231 (1.2577715989339664) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2454 (0.3075020216506673) |
|
| > loader_time: 0.0083 (0.009308677552973167) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:25:48 -- STEP: 10350/30063 -- GLOBAL_STEP: 10350[0m |
|
| > loss_text_ce: 0.05186992138624191 (0.06313835474424018) |
|
| > loss_mel_ce: 4.420424938201904 (4.96585930842709) |
|
| > loss: 1.1180737018585205 (1.2572494158422274) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2853 (0.3075112058344682) |
|
| > loader_time: 0.0087 (0.00930810400829222) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:26:14 -- STEP: 10400/30063 -- GLOBAL_STEP: 10400[0m |
|
| > loss_text_ce: 0.05159060284495354 (0.06305225857569319) |
|
| > loss_mel_ce: 4.625476837158203 (4.9640355559266585) |
|
| > loss: 1.1692668199539185 (1.2567719536790443) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2464 (0.30747214418191293) |
|
| > loader_time: 0.0082 (0.009306314220795262) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:26:39 -- STEP: 10450/30063 -- GLOBAL_STEP: 10450[0m |
|
| > loss_text_ce: 0.04371977597475052 (0.0629651706248663) |
|
| > loss_mel_ce: 4.416383266448975 (4.962114855615721) |
|
| > loss: 1.1150257587432861 (1.2562700066098715) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2537 (0.3073897621392074) |
|
| > loader_time: 0.0087 (0.009304624470797454) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:27:04 -- STEP: 10500/30063 -- GLOBAL_STEP: 10500[0m |
|
| > loss_text_ce: 0.04045282304286957 (0.06288022180256378) |
|
| > loss_mel_ce: 4.312146186828613 (4.959961118561886) |
|
| > loss: 1.0881497859954834 (1.2557103351354564) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2194 (0.3073659356889283) |
|
| > loader_time: 0.0086 (0.009302499612172442) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:27:30 -- STEP: 10550/30063 -- GLOBAL_STEP: 10550[0m |
|
| > loss_text_ce: 0.04451645910739899 (0.06279552242555311) |
|
| > loss_mel_ce: 4.953880310058594 (4.957788552672947) |
|
| > loss: 1.2495992183685303 (1.2551460188259975) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3019 (0.3073851745162542) |
|
| > loader_time: 0.008 (0.009301255370768334) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:27:55 -- STEP: 10600/30063 -- GLOBAL_STEP: 10600[0m |
|
| > loss_text_ce: 0.04350705444812775 (0.06271199698964089) |
|
| > loss_mel_ce: 4.969170093536377 (4.955824810491425) |
|
| > loss: 1.253169298171997 (1.254634201914631) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1848 (0.3073364001400076) |
|
| > loader_time: 0.0083 (0.00929868232529119) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:28:22 -- STEP: 10650/30063 -- GLOBAL_STEP: 10650[0m |
|
| > loss_text_ce: 0.049490541219711304 (0.06263028691527135) |
|
| > loss_mel_ce: 4.5164361000061035 (4.953777034786395) |
|
| > loss: 1.141481637954712 (1.2541018304690474) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3596 (0.3073863224692197) |
|
| > loader_time: 0.0089 (0.009296632775678325) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:28:49 -- STEP: 10700/30063 -- GLOBAL_STEP: 10700[0m |
|
| > loss_text_ce: 0.04379856586456299 (0.06254980638901748) |
|
| > loss_mel_ce: 4.02572774887085 (4.952053799116735) |
|
| > loss: 1.0173815488815308 (1.2536509013844392) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2009 (0.30744275610023647) |
|
| > loader_time: 0.0087 (0.009295135948145497) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:29:14 -- STEP: 10750/30063 -- GLOBAL_STEP: 10750[0m |
|
| > loss_text_ce: 0.041184332221746445 (0.06246910147923354) |
|
| > loss_mel_ce: 4.321810245513916 (4.949850816460552) |
|
| > loss: 1.09074866771698 (1.2530799794973295) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.298 (0.3074132442474375) |
|
| > loader_time: 0.0081 (0.00929274448128634) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:29:39 -- STEP: 10800/30063 -- GLOBAL_STEP: 10800[0m |
|
| > loss_text_ce: 0.05466574430465698 (0.062386307825368706) |
|
| > loss_mel_ce: 4.5749311447143555 (4.947921558177041) |
|
| > loss: 1.1573991775512695 (1.252576966489905) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2888 (0.3073418782817005) |
|
| > loader_time: 0.0085 (0.009290271578011702) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:30:04 -- STEP: 10850/30063 -- GLOBAL_STEP: 10850[0m |
|
| > loss_text_ce: 0.04323854669928551 (0.06230753721660717) |
|
| > loss_mel_ce: 4.261952877044678 (4.946132907340067) |
|
| > loss: 1.0762978792190552 (1.2521101111310948) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2506 (0.30733895710536524) |
|
| > loader_time: 0.0094 (0.00928900901073686) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:30:30 -- STEP: 10900/30063 -- GLOBAL_STEP: 10900[0m |
|
| > loss_text_ce: 0.04208390787243843 (0.06223016297325066) |
|
| > loss_mel_ce: 4.531040668487549 (4.944159147411323) |
|
| > loss: 1.143281102180481 (1.2515973275626453) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2056 (0.30735979819516646) |
|
| > loader_time: 0.009 (0.009287161848960683) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:30:56 -- STEP: 10950/30063 -- GLOBAL_STEP: 10950[0m |
|
| > loss_text_ce: 0.047369055449962616 (0.062152143009857344) |
|
| > loss_mel_ce: 4.727395057678223 (4.942128990556565) |
|
| > loss: 1.1936910152435303 (1.2510702833574099) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3379 (0.3073398395869298) |
|
| > loader_time: 0.0086 (0.009285833171513543) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:31:22 -- STEP: 11000/30063 -- GLOBAL_STEP: 11000[0m |
|
| > loss_text_ce: 0.043431513011455536 (0.06207625814120862) |
|
| > loss_mel_ce: 4.0957818031311035 (4.940389428203769) |
|
| > loss: 1.0348032712936401 (1.25061642155322) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3057 (0.30735996122793807) |
|
| > loader_time: 0.0089 (0.009285420417785654) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:31:50 -- STEP: 11050/30063 -- GLOBAL_STEP: 11050[0m |
|
| > loss_text_ce: 0.04232507199048996 (0.06199860490487696) |
|
| > loss_mel_ce: 4.593900680541992 (4.938755974100737) |
|
| > loss: 1.1590564250946045 (1.250188644725271) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4958 (0.3074452250467716) |
|
| > loader_time: 0.0095 (0.009284549730395848) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:32:16 -- STEP: 11100/30063 -- GLOBAL_STEP: 11100[0m |
|
| > loss_text_ce: 0.03986891359090805 (0.06192052349884484) |
|
| > loss_mel_ce: 3.9813547134399414 (4.9369965221001335) |
|
| > loss: 1.0053058862686157 (1.2497292614036835) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2055 (0.3074579733986048) |
|
| > loader_time: 0.0092 (0.009283515053826416) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:32:42 -- STEP: 11150/30063 -- GLOBAL_STEP: 11150[0m |
|
| > loss_text_ce: 0.04339003562927246 (0.061842740150377204) |
|
| > loss_mel_ce: 4.884430885314941 (4.935050227032663) |
|
| > loss: 1.2319552898406982 (1.2492232418327565) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3042 (0.3074926709273479) |
|
| > loader_time: 0.0081 (0.00928223900730835) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:33:08 -- STEP: 11200/30063 -- GLOBAL_STEP: 11200[0m |
|
| > loss_text_ce: 0.03811433166265488 (0.06176529086760363) |
|
| > loss_mel_ce: 4.602193355560303 (4.933052611351026) |
|
| > loss: 1.1600769758224487 (1.2487044755422632) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.225 (0.3075171173257497) |
|
| > loader_time: 0.0092 (0.009280633181333538) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:33:35 -- STEP: 11250/30063 -- GLOBAL_STEP: 11250[0m |
|
| > loss_text_ce: 0.04009737819433212 (0.06168698878122685) |
|
| > loss_mel_ce: 4.639787673950195 (4.931229447703905) |
|
| > loss: 1.169971227645874 (1.2482291091018243) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3121 (0.30753642082214466) |
|
| > loader_time: 0.0087 (0.009278857188754595) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:34:00 -- STEP: 11300/30063 -- GLOBAL_STEP: 11300[0m |
|
| > loss_text_ce: 0.04528151825070381 (0.06161454172425814) |
|
| > loss_mel_ce: 3.8009588718414307 (4.929205906116871) |
|
| > loss: 0.961560070514679 (1.2477051119097553) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1876 (0.3074960428634584) |
|
| > loader_time: 0.0083 (0.009277473487685197) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:34:26 -- STEP: 11350/30063 -- GLOBAL_STEP: 11350[0m |
|
| > loss_text_ce: 0.04788221791386604 (0.06154117748042808) |
|
| > loss_mel_ce: 4.789422988891602 (4.927349509310635) |
|
| > loss: 1.2093262672424316 (1.2472226716505794) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3024 (0.30753151282339986) |
|
| > loader_time: 0.0093 (0.009276164155699599) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:34:52 -- STEP: 11400/30063 -- GLOBAL_STEP: 11400[0m |
|
| > loss_text_ce: 0.043647926300764084 (0.06146557947457345) |
|
| > loss_mel_ce: 4.334468841552734 (4.925313280423499) |
|
| > loss: 1.094529151916504 (1.2466947149655263) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2396 (0.3074915554021542) |
|
| > loader_time: 0.0096 (0.009274755151648257) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:35:18 -- STEP: 11450/30063 -- GLOBAL_STEP: 11450[0m |
|
| > loss_text_ce: 0.04008049517869949 (0.061390320308278595) |
|
| > loss_mel_ce: 4.669530391693115 (4.9232789974546005) |
|
| > loss: 1.1774027347564697 (1.2461673294352642) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3743 (0.30749865681844096) |
|
| > loader_time: 0.0104 (0.009274467380806855) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:35:44 -- STEP: 11500/30063 -- GLOBAL_STEP: 11500[0m |
|
| > loss_text_ce: 0.04463668540120125 (0.061317288802369634) |
|
| > loss_mel_ce: 4.804383277893066 (4.921257623050545) |
|
| > loss: 1.2122550010681152 (1.2456437279452441) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2746 (0.3074950084686288) |
|
| > loader_time: 0.0086 (0.009272557196409797) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:36:08 -- STEP: 11550/30063 -- GLOBAL_STEP: 11550[0m |
|
| > loss_text_ce: 0.04534080997109413 (0.061246064917801116) |
|
| > loss_mel_ce: 4.371310710906982 (4.919232475499592) |
|
| > loss: 1.1041629314422607 (1.2451196350989393) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2882 (0.3074358258722154) |
|
| > loader_time: 0.0088 (0.009270697011576063) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:36:34 -- STEP: 11600/30063 -- GLOBAL_STEP: 11600[0m |
|
| > loss_text_ce: 0.040644340217113495 (0.06117405682127779) |
|
| > loss_mel_ce: 4.364027976989746 (4.917290277255015) |
|
| > loss: 1.1011680364608765 (1.2446160835079068) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2374 (0.30742867319748385) |
|
| > loader_time: 0.0087 (0.00926907781896919) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:37:00 -- STEP: 11650/30063 -- GLOBAL_STEP: 11650[0m |
|
| > loss_text_ce: 0.04068606719374657 (0.06110194205086616) |
|
| > loss_mel_ce: 4.3269362449646 (4.915155055840151) |
|
| > loss: 1.0919055938720703 (1.2440642494807403) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1915 (0.30742117179821565) |
|
| > loader_time: 0.0078 (0.009267638926853939) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:37:26 -- STEP: 11700/30063 -- GLOBAL_STEP: 11700[0m |
|
| > loss_text_ce: 0.04523228853940964 (0.0610304809068766) |
|
| > loss_mel_ce: 4.482456207275391 (4.913419318708618) |
|
| > loss: 1.1319221258163452 (1.2436124498976584) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2734 (0.30745505648800525) |
|
| > loader_time: 0.0091 (0.00926680880734044) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:37:52 -- STEP: 11750/30063 -- GLOBAL_STEP: 11750[0m |
|
| > loss_text_ce: 0.04164936766028404 (0.060957539411935255) |
|
| > loss_mel_ce: 4.371998310089111 (4.9112012915712935) |
|
| > loss: 1.1034119129180908 (1.2430397077266193) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3082 (0.3074599346404391) |
|
| > loader_time: 0.0084 (0.00926557502340764) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:38:18 -- STEP: 11800/30063 -- GLOBAL_STEP: 11800[0m |
|
| > loss_text_ce: 0.046756986528635025 (0.060887990358080256) |
|
| > loss_mel_ce: 4.79047155380249 (4.9093842670069066) |
|
| > loss: 1.2093071937561035 (1.2425680643168528) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1735 (0.3074478529469451) |
|
| > loader_time: 0.0085 (0.009264048014656988) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:38:45 -- STEP: 11850/30063 -- GLOBAL_STEP: 11850[0m |
|
| > loss_text_ce: 0.04690614715218544 (0.06081919361644642) |
|
| > loss_mel_ce: 3.9194750785827637 (4.907402724475317) |
|
| > loss: 0.9915953278541565 (1.2420554794691765) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3445 (0.30751572146194484) |
|
| > loader_time: 0.0091 (0.00926319428134067) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:39:12 -- STEP: 11900/30063 -- GLOBAL_STEP: 11900[0m |
|
| > loss_text_ce: 0.043324876576662064 (0.06075021928943241) |
|
| > loss_mel_ce: 4.400256633758545 (4.905776227041471) |
|
| > loss: 1.1108953952789307 (1.2416316115505526) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2313 (0.30754688747790737) |
|
| > loader_time: 0.0081 (0.00926224516219453) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:39:37 -- STEP: 11950/30063 -- GLOBAL_STEP: 11950[0m |
|
| > loss_text_ce: 0.04530574753880501 (0.06068029705263567) |
|
| > loss_mel_ce: 4.623571872711182 (4.90410553742654) |
|
| > loss: 1.1672194004058838 (1.241196458604043) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2508 (0.3075305535504014) |
|
| > loader_time: 0.009 (0.009261040607755663) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:40:04 -- STEP: 12000/30063 -- GLOBAL_STEP: 12000[0m |
|
| > loss_text_ce: 0.04893041402101517 (0.06061391102833038) |
|
| > loss_mel_ce: 4.919193267822266 (4.902676748355251) |
|
| > loss: 1.2420309782028198 (1.2408226648469738) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2711 (0.30758818437655805) |
|
| > loader_time: 0.0088 (0.009259514431158713) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_12000.pth |
|
|
|
[1m --> TIME: 2024-10-26 09:40:44 -- STEP: 12050/30063 -- GLOBAL_STEP: 12050[0m |
|
| > loss_text_ce: 0.043795425444841385 (0.060545098793333356) |
|
| > loss_mel_ce: 4.284984588623047 (4.900787907082038) |
|
| > loss: 1.0821950435638428 (1.2403332514931025) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3613 (0.30757709479430934) |
|
| > loader_time: 0.0096 (0.00925977006492775) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:41:09 -- STEP: 12100/30063 -- GLOBAL_STEP: 12100[0m |
|
| > loss_text_ce: 0.04468939080834389 (0.06047493495693616) |
|
| > loss_mel_ce: 4.306922912597656 (4.8989265521893035) |
|
| > loss: 1.0879030227661133 (1.2398503718248053) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2095 (0.30754347677073235) |
|
| > loader_time: 0.0085 (0.00925852235683727) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:41:35 -- STEP: 12150/30063 -- GLOBAL_STEP: 12150[0m |
|
| > loss_text_ce: 0.048856157809495926 (0.060406149037396814) |
|
| > loss_mel_ce: 4.719479084014893 (4.896998180931017) |
|
| > loss: 1.1920838356018066 (1.2393510825202263) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2949 (0.3075311496718921) |
|
| > loader_time: 0.0083 (0.009256773250093213) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:42:01 -- STEP: 12200/30063 -- GLOBAL_STEP: 12200[0m |
|
| > loss_text_ce: 0.046400874853134155 (0.060341220195786735) |
|
| > loss_mel_ce: 4.550384044647217 (4.895332270215788) |
|
| > loss: 1.1491962671279907 (1.2389183726564752) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.212 (0.30752775452176084) |
|
| > loader_time: 0.009 (0.00925582199800213) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:42:26 -- STEP: 12250/30063 -- GLOBAL_STEP: 12250[0m |
|
| > loss_text_ce: 0.04334597662091255 (0.06027286043732736) |
|
| > loss_mel_ce: 4.602388381958008 (4.893202876635976) |
|
| > loss: 1.1614335775375366 (1.2383689343053434) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.377 (0.3074975147247315) |
|
| > loader_time: 0.0089 (0.009253975459507562) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:42:52 -- STEP: 12300/30063 -- GLOBAL_STEP: 12300[0m |
|
| > loss_text_ce: 0.04300916567444801 (0.0602083785968218) |
|
| > loss_mel_ce: 4.4165449142456055 (4.8914678343718805) |
|
| > loss: 1.1148885488510132 (1.2379190532899456) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.225 (0.3074895622865943) |
|
| > loader_time: 0.0092 (0.009253245330438408) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:43:18 -- STEP: 12350/30063 -- GLOBAL_STEP: 12350[0m |
|
| > loss_text_ce: 0.038267143070697784 (0.060140637354510376) |
|
| > loss_mel_ce: 4.305757522583008 (4.889687799712446) |
|
| > loss: 1.0860061645507812 (1.2374571093180848) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2836 (0.30750177989604527) |
|
| > loader_time: 0.0099 (0.009252584144654092) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:43:44 -- STEP: 12400/30063 -- GLOBAL_STEP: 12400[0m |
|
| > loss_text_ce: 0.04278199002146721 (0.06007430747241489) |
|
| > loss_mel_ce: 4.11808967590332 (4.887955180387369) |
|
| > loss: 1.0402178764343262 (1.2370073720043675) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2287 (0.30753053536338215) |
|
| > loader_time: 0.0085 (0.009251727942497526) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:44:10 -- STEP: 12450/30063 -- GLOBAL_STEP: 12450[0m |
|
| > loss_text_ce: 0.04406498745083809 (0.06000631342929995) |
|
| > loss_mel_ce: 4.708452224731445 (4.886060203452688) |
|
| > loss: 1.1881293058395386 (1.2365166292994823) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3006 (0.3075280897875869) |
|
| > loader_time: 0.0085 (0.009250940242445639) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:44:37 -- STEP: 12500/30063 -- GLOBAL_STEP: 12500[0m |
|
| > loss_text_ce: 0.04377176985144615 (0.05994089479774223) |
|
| > loss_mel_ce: 4.8808465003967285 (4.884314359798452) |
|
| > loss: 1.2311545610427856 (1.2360638137006719) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2081 (0.30759490468978906) |
|
| > loader_time: 0.009 (0.009250369892120375) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:45:02 -- STEP: 12550/30063 -- GLOBAL_STEP: 12550[0m |
|
| > loss_text_ce: 0.04716948792338371 (0.0598782557771499) |
|
| > loss_mel_ce: 4.0573201179504395 (4.882644639110207) |
|
| > loss: 1.0261224508285522 (1.2356307237604176) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3625 (0.30754963441673994) |
|
| > loader_time: 0.0093 (0.009248658309419809) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:45:27 -- STEP: 12600/30063 -- GLOBAL_STEP: 12600[0m |
|
| > loss_text_ce: 0.0436917282640934 (0.05981297798602577) |
|
| > loss_mel_ce: 4.382955074310303 (4.880552474127899) |
|
| > loss: 1.1066616773605347 (1.235091363073337) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2528 (0.30748113743842603) |
|
| > loader_time: 0.0084 (0.009247208701239705) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:45:52 -- STEP: 12650/30063 -- GLOBAL_STEP: 12650[0m |
|
| > loss_text_ce: 0.03802647069096565 (0.05974712433521678) |
|
| > loss_mel_ce: 3.9960849285125732 (4.878706712722799) |
|
| > loss: 1.0085278749465942 (1.2346134593269995) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3047 (0.3074466373420984) |
|
| > loader_time: 0.0089 (0.009247009235879658) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:46:17 -- STEP: 12700/30063 -- GLOBAL_STEP: 12700[0m |
|
| > loss_text_ce: 0.04801427572965622 (0.059683691560635285) |
|
| > loss_mel_ce: 4.893192291259766 (4.8768257044619645) |
|
| > loss: 1.2353016138076782 (1.2341273490368811) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3458 (0.3074071559192625) |
|
| > loader_time: 0.0097 (0.009245572371745689) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:46:43 -- STEP: 12750/30063 -- GLOBAL_STEP: 12750[0m |
|
| > loss_text_ce: 0.04432092607021332 (0.059622032502994686) |
|
| > loss_mel_ce: 4.735030174255371 (4.875061599881058) |
|
| > loss: 1.1948378086090088 (1.2336709081144854) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.302 (0.30740584324855397) |
|
| > loader_time: 0.0096 (0.00924455132203945) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:47:10 -- STEP: 12800/30063 -- GLOBAL_STEP: 12800[0m |
|
| > loss_text_ce: 0.04382703825831413 (0.05955506492464324) |
|
| > loss_mel_ce: 4.636072158813477 (4.8732372653111975) |
|
| > loss: 1.1699748039245605 (1.23319808259606) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2484 (0.3074237504228949) |
|
| > loader_time: 0.0085 (0.00924404272809626) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:47:37 -- STEP: 12850/30063 -- GLOBAL_STEP: 12850[0m |
|
| > loss_text_ce: 0.042145732790231705 (0.059492391728531394) |
|
| > loss_mel_ce: 4.935103893280029 (4.8718144065767985) |
|
| > loss: 1.2443124055862427 (1.2328266996214794) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.357 (0.3074965721902213) |
|
| > loader_time: 0.0088 (0.009242973197758901) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:48:03 -- STEP: 12900/30063 -- GLOBAL_STEP: 12900[0m |
|
| > loss_text_ce: 0.04121166095137596 (0.05942995931346736) |
|
| > loss_mel_ce: 4.287423133850098 (4.8699677766755665) |
|
| > loss: 1.0821586847305298 (1.2323494340329175) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2094 (0.30754117572030326) |
|
| > loader_time: 0.0086 (0.009243348679801299) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:48:28 -- STEP: 12950/30063 -- GLOBAL_STEP: 12950[0m |
|
| > loss_text_ce: 0.043676894158124924 (0.059366723780581256) |
|
| > loss_mel_ce: 4.128334045410156 (4.868060077387406) |
|
| > loss: 1.043002724647522 (1.2318567003200385) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3441 (0.30747406703624913) |
|
| > loader_time: 0.0092 (0.009241971675033283) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:48:54 -- STEP: 13000/30063 -- GLOBAL_STEP: 13000[0m |
|
| > loss_text_ce: 0.04051762819290161 (0.05930487494399901) |
|
| > loss_mel_ce: 4.365699768066406 (4.866051356774119) |
|
| > loss: 1.1015543937683105 (1.2313390579544556) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.288 (0.3074841856222882) |
|
| > loader_time: 0.0084 (0.009240871612842268) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:49:21 -- STEP: 13050/30063 -- GLOBAL_STEP: 13050[0m |
|
| > loss_text_ce: 0.04123494029045105 (0.05924402801155802) |
|
| > loss_mel_ce: 4.169182777404785 (4.864193515448742) |
|
| > loss: 1.0526044368743896 (1.230859385884125) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3076 (0.30753996158468266) |
|
| > loader_time: 0.0087 (0.009240716188803486) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:49:46 -- STEP: 13100/30063 -- GLOBAL_STEP: 13100[0m |
|
| > loss_text_ce: 0.03467089682817459 (0.0591848131260684) |
|
| > loss_mel_ce: 4.013933181762695 (4.862415088515247) |
|
| > loss: 1.0121510028839111 (1.230399975435422) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2244 (0.30751952515303593) |
|
| > loader_time: 0.0086 (0.009239344924460838) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:50:13 -- STEP: 13150/30063 -- GLOBAL_STEP: 13150[0m |
|
| > loss_text_ce: 0.03786170855164528 (0.05912429289616093) |
|
| > loss_mel_ce: 4.219997882843018 (4.860934923918981) |
|
| > loss: 1.0644649267196655 (1.2300148042481183) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2625 (0.3075352037814177) |
|
| > loader_time: 0.0086 (0.009238131780588126) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:50:37 -- STEP: 13200/30063 -- GLOBAL_STEP: 13200[0m |
|
| > loss_text_ce: 0.05002226307988167 (0.059061273262186804) |
|
| > loss_mel_ce: 4.74195671081543 (4.859009072347115) |
|
| > loss: 1.1979947090148926 (1.2295175864795822) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2514 (0.3074969845287722) |
|
| > loader_time: 0.009 (0.009237328713590448) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:51:03 -- STEP: 13250/30063 -- GLOBAL_STEP: 13250[0m |
|
| > loss_text_ce: 0.043223559856414795 (0.05900275300190121) |
|
| > loss_mel_ce: 4.613772869110107 (4.857439465900627) |
|
| > loss: 1.164249062538147 (1.2291105548075885) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3609 (0.3074571626591225) |
|
| > loader_time: 0.0094 (0.009236763630273205) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:51:27 -- STEP: 13300/30063 -- GLOBAL_STEP: 13300[0m |
|
| > loss_text_ce: 0.0421791598200798 (0.05894330901927057) |
|
| > loss_mel_ce: 4.332854270935059 (4.855741078889451) |
|
| > loss: 1.0937583446502686 (1.2286710970742345) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2006 (0.3073946033384561) |
|
| > loader_time: 0.0101 (0.009236229559532677) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:51:52 -- STEP: 13350/30063 -- GLOBAL_STEP: 13350[0m |
|
| > loss_text_ce: 0.041880011558532715 (0.058883763881825646) |
|
| > loss_mel_ce: 4.741331100463867 (4.854050355118304) |
|
| > loss: 1.1958028078079224 (1.2282335298740008) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4017 (0.3073310096612129) |
|
| > loader_time: 0.0097 (0.009235905493689833) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:52:18 -- STEP: 13400/30063 -- GLOBAL_STEP: 13400[0m |
|
| > loss_text_ce: 0.04548204317688942 (0.05882664655015527) |
|
| > loss_mel_ce: 4.100371360778809 (4.852021604562865) |
|
| > loss: 1.0364633798599243 (1.2277120629219858) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3597 (0.30735853084877274) |
|
| > loader_time: 0.0093 (0.00923471872486285) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:52:44 -- STEP: 13450/30063 -- GLOBAL_STEP: 13450[0m |
|
| > loss_text_ce: 0.045175231993198395 (0.05876886596605446) |
|
| > loss_mel_ce: 4.088685989379883 (4.850362415792318) |
|
| > loss: 1.0334652662277222 (1.2272828205810595) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3763 (0.3073411065849665) |
|
| > loader_time: 0.0089 (0.009233767871076722) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:53:09 -- STEP: 13500/30063 -- GLOBAL_STEP: 13500[0m |
|
| > loss_text_ce: 0.04127155616879463 (0.05871015053711543) |
|
| > loss_mel_ce: 4.1588616371154785 (4.848365241297957) |
|
| > loss: 1.0500333309173584 (1.2267688481012973) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2473 (0.3073092738610719) |
|
| > loader_time: 0.0098 (0.009233747694227417) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:53:34 -- STEP: 13550/30063 -- GLOBAL_STEP: 13550[0m |
|
| > loss_text_ce: 0.04087664186954498 (0.05865213444339601) |
|
| > loss_mel_ce: 4.036096572875977 (4.846633702908042) |
|
| > loss: 1.0192433595657349 (1.2263214595115484) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4447 (0.3072902928947075) |
|
| > loader_time: 0.0112 (0.009232732202733995) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:54:00 -- STEP: 13600/30063 -- GLOBAL_STEP: 13600[0m |
|
| > loss_text_ce: 0.04948098957538605 (0.05859549955218361) |
|
| > loss_mel_ce: 5.147891521453857 (4.845229378710762) |
|
| > loss: 1.2993431091308594 (1.2259562197152298) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3242 (0.3072888804533894) |
|
| > loader_time: 0.0097 (0.009232706112020131) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:54:26 -- STEP: 13650/30063 -- GLOBAL_STEP: 13650[0m |
|
| > loss_text_ce: 0.03701704740524292 (0.05853785963367399) |
|
| > loss_mel_ce: 4.636731147766113 (4.843632612175981) |
|
| > loss: 1.1684370040893555 (1.2255426181061355) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2117 (0.30730121244004277) |
|
| > loader_time: 0.0092 (0.009231516223250701) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:54:52 -- STEP: 13700/30063 -- GLOBAL_STEP: 13700[0m |
|
| > loss_text_ce: 0.043017659336328506 (0.05848347646231839) |
|
| > loss_mel_ce: 4.437896728515625 (4.841885631989376) |
|
| > loss: 1.12022864818573 (1.2250922772397081) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2886 (0.30731421388848773) |
|
| > loader_time: 0.0089 (0.009230976783446124) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:55:18 -- STEP: 13750/30063 -- GLOBAL_STEP: 13750[0m |
|
| > loss_text_ce: 0.042664822190999985 (0.058423461503332184) |
|
| > loss_mel_ce: 4.523960113525391 (4.839983669315672) |
|
| > loss: 1.1416562795639038 (1.2246017828334455) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4673 (0.3073263221393924) |
|
| > loader_time: 0.0093 (0.009229874264110195) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:55:45 -- STEP: 13800/30063 -- GLOBAL_STEP: 13800[0m |
|
| > loss_text_ce: 0.047376859933137894 (0.058364842705752415) |
|
| > loss_mel_ce: 5.001344203948975 (4.838097296503994) |
|
| > loss: 1.262180209159851 (1.2241155349171673) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3252 (0.3073434847334149) |
|
| > loader_time: 0.0112 (0.009230145630629141) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:56:11 -- STEP: 13850/30063 -- GLOBAL_STEP: 13850[0m |
|
| > loss_text_ce: 0.044620346277952194 (0.058311647763105816) |
|
| > loss_mel_ce: 4.8541388511657715 (4.836692869430832) |
|
| > loss: 1.2246898412704468 (1.2237511294284014) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3578 (0.3073566392526722) |
|
| > loader_time: 0.0089 (0.009229391679867057) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:56:35 -- STEP: 13900/30063 -- GLOBAL_STEP: 13900[0m |
|
| > loss_text_ce: 0.040659502148628235 (0.058257095829295474) |
|
| > loss_mel_ce: 4.195016860961914 (4.834990588915422) |
|
| > loss: 1.058919072151184 (1.2233119213023624) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2316 (0.30730354104968205) |
|
| > loader_time: 0.0089 (0.009228317240159266) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:57:02 -- STEP: 13950/30063 -- GLOBAL_STEP: 13950[0m |
|
| > loss_text_ce: 0.049711406230926514 (0.05820385539403514) |
|
| > loss_mel_ce: 4.340839385986328 (4.833771834048734) |
|
| > loss: 1.09763765335083 (1.2229939224771267) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.398 (0.30734325020971187) |
|
| > loader_time: 0.0096 (0.009227300869521253) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:57:28 -- STEP: 14000/30063 -- GLOBAL_STEP: 14000[0m |
|
| > loss_text_ce: 0.039197709411382675 (0.05814947410140706) |
|
| > loss_mel_ce: 3.950216770172119 (4.832381835818278) |
|
| > loss: 0.9973536133766174 (1.2226328276055192) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.243 (0.30736607020241796) |
|
| > loader_time: 0.0095 (0.009226345351764111) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_14000.pth |
|
|
|
[1m --> TIME: 2024-10-26 09:58:08 -- STEP: 14050/30063 -- GLOBAL_STEP: 14050[0m |
|
| > loss_text_ce: 0.0439474917948246 (0.0580960465885459) |
|
| > loss_mel_ce: 3.862816572189331 (4.8305715925413395) |
|
| > loss: 0.9766910076141357 (1.2221669099296957) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2689 (0.30736765941266836) |
|
| > loader_time: 0.009 (0.009226475304980273) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:58:34 -- STEP: 14100/30063 -- GLOBAL_STEP: 14100[0m |
|
| > loss_text_ce: 0.04417821392416954 (0.058045504410216026) |
|
| > loss_mel_ce: 4.249610900878906 (4.828725071764994) |
|
| > loss: 1.0734472274780273 (1.2216926442308622) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3037 (0.30735278844833297) |
|
| > loader_time: 0.0088 (0.009225653807322163) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:58:58 -- STEP: 14150/30063 -- GLOBAL_STEP: 14150[0m |
|
| > loss_text_ce: 0.04187445715069771 (0.057992581342850354) |
|
| > loss_mel_ce: 4.215877056121826 (4.826877176197161) |
|
| > loss: 1.0644378662109375 (1.2212174395790363) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.302 (0.30730186063072207) |
|
| > loader_time: 0.0084 (0.009224979801649739) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:59:23 -- STEP: 14200/30063 -- GLOBAL_STEP: 14200[0m |
|
| > loss_text_ce: 0.044174693524837494 (0.0579418930065043) |
|
| > loss_mel_ce: 4.8073506355285645 (4.825386373761662) |
|
| > loss: 1.212881326675415 (1.2208320668717498) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3479 (0.30725603123785666) |
|
| > loader_time: 0.0099 (0.009224662377800718) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 09:59:50 -- STEP: 14250/30063 -- GLOBAL_STEP: 14250[0m |
|
| > loss_text_ce: 0.04539482668042183 (0.057888460299686265) |
|
| > loss_mel_ce: 4.5705952644348145 (4.823584514902337) |
|
| > loss: 1.153997540473938 (1.2203682440038306) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3375 (0.30727939169030366) |
|
| > loader_time: 0.0086 (0.009225165283470801) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:00:15 -- STEP: 14300/30063 -- GLOBAL_STEP: 14300[0m |
|
| > loss_text_ce: 0.04202690348029137 (0.05783264448254038) |
|
| > loss_mel_ce: 4.775210857391357 (4.821592292418835) |
|
| > loss: 1.2043094635009766 (1.2198562344304325) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2728 (0.3072528399787575) |
|
| > loader_time: 0.0087 (0.009224130056954742) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:00:40 -- STEP: 14350/30063 -- GLOBAL_STEP: 14350[0m |
|
| > loss_text_ce: 0.04136889800429344 (0.057778874847531615) |
|
| > loss_mel_ce: 4.076894283294678 (4.8201054398274055) |
|
| > loss: 1.0295658111572266 (1.219471078899263) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2637 (0.3072275838951608) |
|
| > loader_time: 0.0083 (0.009222847759100596) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:01:05 -- STEP: 14400/30063 -- GLOBAL_STEP: 14400[0m |
|
| > loss_text_ce: 0.041974857449531555 (0.05772288659044219) |
|
| > loss_mel_ce: 4.497849464416504 (4.818337837424531) |
|
| > loss: 1.1349561214447021 (1.2190151812260341) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2724 (0.30719146516587925) |
|
| > loader_time: 0.0091 (0.009222031086683251) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:01:30 -- STEP: 14450/30063 -- GLOBAL_STEP: 14450[0m |
|
| > loss_text_ce: 0.038760215044021606 (0.05766856144404729) |
|
| > loss_mel_ce: 4.33547830581665 (4.816477950469008) |
|
| > loss: 1.0935596227645874 (1.2185366282042327) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2294 (0.30714164737186556) |
|
| > loader_time: 0.0082 (0.009221284051254938) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:01:56 -- STEP: 14500/30063 -- GLOBAL_STEP: 14500[0m |
|
| > loss_text_ce: 0.0442153736948967 (0.057619747874808606) |
|
| > loss_mel_ce: 3.9625165462493896 (4.815100455086795) |
|
| > loss: 1.001682996749878 (1.2181800509855658) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1868 (0.30715769040995555) |
|
| > loader_time: 0.0082 (0.009219908928049) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:02:23 -- STEP: 14550/30063 -- GLOBAL_STEP: 14550[0m |
|
| > loss_text_ce: 0.041927270591259 (0.05756679781190911) |
|
| > loss_mel_ce: 3.902470111846924 (4.813365619993689) |
|
| > loss: 0.986099362373352 (1.2177331047041713) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3383 (0.30719332216531475) |
|
| > loader_time: 0.0092 (0.009219135107453315) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:02:48 -- STEP: 14600/30063 -- GLOBAL_STEP: 14600[0m |
|
| > loss_text_ce: 0.04428572580218315 (0.05751727456558637) |
|
| > loss_mel_ce: 4.564116477966309 (4.811764474241689) |
|
| > loss: 1.1521005630493164 (1.2173204374476647) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3592 (0.3071636829800794) |
|
| > loader_time: 0.0097 (0.009219111890008983) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:03:13 -- STEP: 14650/30063 -- GLOBAL_STEP: 14650[0m |
|
| > loss_text_ce: 0.03969542309641838 (0.057468018506624884) |
|
| > loss_mel_ce: 4.051859378814697 (4.8105880610039495) |
|
| > loss: 1.0228886604309082 (1.2170140201199169) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2336 (0.3071296087466809) |
|
| > loader_time: 0.0088 (0.009217660679345223) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:03:40 -- STEP: 14700/30063 -- GLOBAL_STEP: 14700[0m |
|
| > loss_text_ce: 0.04381052032113075 (0.05741681098836608) |
|
| > loss_mel_ce: 3.815000534057617 (4.8089970258141745) |
|
| > loss: 0.9647027850151062 (1.2166034594320108) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2095 (0.3071716681467425) |
|
| > loader_time: 0.0088 (0.009217097467305688) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:04:05 -- STEP: 14750/30063 -- GLOBAL_STEP: 14750[0m |
|
| > loss_text_ce: 0.04406379908323288 (0.057365661565782544) |
|
| > loss_mel_ce: 3.791686534881592 (4.807168190018594) |
|
| > loss: 0.9589375853538513 (1.2161334631281377) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2323 (0.30714761995865053) |
|
| > loader_time: 0.0086 (0.00921638509782692) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:04:31 -- STEP: 14800/30063 -- GLOBAL_STEP: 14800[0m |
|
| > loss_text_ce: 0.04454737529158592 (0.05731797180667105) |
|
| > loss_mel_ce: 4.419222354888916 (4.805535913738033) |
|
| > loss: 1.1159424781799316 (1.2157134716212743) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1883 (0.30714284146154175) |
|
| > loader_time: 0.0084 (0.009215343820082154) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:04:56 -- STEP: 14850/30063 -- GLOBAL_STEP: 14850[0m |
|
| > loss_text_ce: 0.04080287739634514 (0.057269140851467434) |
|
| > loss_mel_ce: 4.3369011878967285 (4.804428064285292) |
|
| > loss: 1.0944260358810425 (1.215424301519939) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3573 (0.30711796161702914) |
|
| > loader_time: 0.0095 (0.009214752161944333) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:05:23 -- STEP: 14900/30063 -- GLOBAL_STEP: 14900[0m |
|
| > loss_text_ce: 0.04454677924513817 (0.057218162164391975) |
|
| > loss_mel_ce: 4.551098823547363 (4.802946955773643) |
|
| > loss: 1.1489113569259644 (1.2150412797087784) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2084 (0.3071756732703848) |
|
| > loader_time: 0.0093 (0.009214156426039297) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:05:48 -- STEP: 14950/30063 -- GLOBAL_STEP: 14950[0m |
|
| > loss_text_ce: 0.04580620676279068 (0.057167346125661614) |
|
| > loss_mel_ce: 4.140129089355469 (4.801518270323496) |
|
| > loss: 1.0464838743209839 (1.2146714043378026) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3289 (0.3071339155599019) |
|
| > loader_time: 0.009 (0.009213535618223852) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:06:14 -- STEP: 15000/30063 -- GLOBAL_STEP: 15000[0m |
|
| > loss_text_ce: 0.040005140006542206 (0.057116303834070646) |
|
| > loss_mel_ce: 4.481742858886719 (4.8000050004164265) |
|
| > loss: 1.1304370164871216 (1.2142803263068191) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3459 (0.3071283787091566) |
|
| > loader_time: 0.0095 (0.009213288338979064) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:06:40 -- STEP: 15050/30063 -- GLOBAL_STEP: 15050[0m |
|
| > loss_text_ce: 0.040979478508234024 (0.05706607895510328) |
|
| > loss_mel_ce: 4.947078704833984 (4.798434780824213) |
|
| > loss: 1.2470145225524902 (1.213875215192173) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5219 (0.30713194360764645) |
|
| > loader_time: 0.0098 (0.009212160268891273) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:07:05 -- STEP: 15100/30063 -- GLOBAL_STEP: 15100[0m |
|
| > loss_text_ce: 0.03658517450094223 (0.057017292514295256) |
|
| > loss_mel_ce: 3.9633615016937256 (4.796549914751609) |
|
| > loss: 0.9999866485595703 (1.2133918020504193) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2021 (0.30710806890828646) |
|
| > loader_time: 0.0081 (0.00921161286878268) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:07:32 -- STEP: 15150/30063 -- GLOBAL_STEP: 15150[0m |
|
| > loss_text_ce: 0.040347617119550705 (0.05696849400503396) |
|
| > loss_mel_ce: 4.799482822418213 (4.795164297862403) |
|
| > loss: 1.2099575996398926 (1.2130331981772235) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2545 (0.3071371274576715) |
|
| > loader_time: 0.0084 (0.009211009390676749) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:07:57 -- STEP: 15200/30063 -- GLOBAL_STEP: 15200[0m |
|
| > loss_text_ce: 0.04689180850982666 (0.05692199065178426) |
|
| > loss_mel_ce: 4.483241558074951 (4.793462210407371) |
|
| > loss: 1.132533311843872 (1.212596050500869) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3748 (0.30712282279604286) |
|
| > loader_time: 0.01 (0.009210360552135246) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:08:24 -- STEP: 15250/30063 -- GLOBAL_STEP: 15250[0m |
|
| > loss_text_ce: 0.038457710295915604 (0.05687278647256668) |
|
| > loss_mel_ce: 3.7906277179718018 (4.791401154580652) |
|
| > loss: 0.9572713375091553 (1.2120684854945192) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.255 (0.30716123544974333) |
|
| > loader_time: 0.0091 (0.00920964497425515) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:08:50 -- STEP: 15300/30063 -- GLOBAL_STEP: 15300[0m |
|
| > loss_text_ce: 0.046418625861406326 (0.056826051765066624) |
|
| > loss_mel_ce: 4.128140926361084 (4.78959283351897) |
|
| > loss: 1.043639898300171 (1.2116047215422767) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2714 (0.3071522229948847) |
|
| > loader_time: 0.01 (0.009210147203183621) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:09:15 -- STEP: 15350/30063 -- GLOBAL_STEP: 15350[0m |
|
| > loss_text_ce: 0.036800842732191086 (0.05677706609719728) |
|
| > loss_mel_ce: 4.052865028381348 (4.788072031437369) |
|
| > loss: 1.0224164724349976 (1.211212274612743) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2594 (0.3071468064994681) |
|
| > loader_time: 0.0084 (0.009210738794034758) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:09:42 -- STEP: 15400/30063 -- GLOBAL_STEP: 15400[0m |
|
| > loss_text_ce: 0.047413863241672516 (0.056729931444074705) |
|
| > loss_mel_ce: 4.3706769943237305 (4.786643409140687) |
|
| > loss: 1.104522705078125 (1.2108433353707382) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1744 (0.307163933856146) |
|
| > loader_time: 0.0092 (0.009210683246711612) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:10:09 -- STEP: 15450/30063 -- GLOBAL_STEP: 15450[0m |
|
| > loss_text_ce: 0.052413519471883774 (0.05668340838963342) |
|
| > loss_mel_ce: 4.598607063293457 (4.7851808600903984) |
|
| > loss: 1.1627551317214966 (1.2104660673357515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3778 (0.30723696580596654) |
|
| > loader_time: 0.0087 (0.009210292992082593) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:10:35 -- STEP: 15500/30063 -- GLOBAL_STEP: 15500[0m |
|
| > loss_text_ce: 0.04548536613583565 (0.056636127450773684) |
|
| > loss_mel_ce: 4.624387264251709 (4.783980648948289) |
|
| > loss: 1.1674681901931763 (1.2101541943050194) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2097 (0.3072244505882256) |
|
| > loader_time: 0.0087 (0.009209642456423832) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:11:01 -- STEP: 15550/30063 -- GLOBAL_STEP: 15550[0m |
|
| > loss_text_ce: 0.044909313321113586 (0.05659098759389373) |
|
| > loss_mel_ce: 4.497960090637207 (4.782339907735095) |
|
| > loss: 1.1357173919677734 (1.2097327240295341) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2695 (0.3072233274588624) |
|
| > loader_time: 0.009 (0.009209625897300185) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:11:27 -- STEP: 15600/30063 -- GLOBAL_STEP: 15600[0m |
|
| > loss_text_ce: 0.039099402725696564 (0.056543005934605886) |
|
| > loss_mel_ce: 5.13686466217041 (4.780542155152701) |
|
| > loss: 1.293990969657898 (1.2092712904703915) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2529 (0.3072501161006774) |
|
| > loader_time: 0.0087 (0.009208962886761375) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:11:52 -- STEP: 15650/30063 -- GLOBAL_STEP: 15650[0m |
|
| > loss_text_ce: 0.040448885411024094 (0.05649542203655057) |
|
| > loss_mel_ce: 4.7154340744018555 (4.7787382614574385) |
|
| > loss: 1.188970685005188 (1.208808421064108) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4368 (0.3071940990911117) |
|
| > loader_time: 0.0097 (0.009208658952682525) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:12:18 -- STEP: 15700/30063 -- GLOBAL_STEP: 15700[0m |
|
| > loss_text_ce: 0.0488094799220562 (0.056451956418574124) |
|
| > loss_mel_ce: 4.462857246398926 (4.777311706937791) |
|
| > loss: 1.127916693687439 (1.2084409160333067) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2094 (0.3072000407717024) |
|
| > loader_time: 0.0082 (0.009208221602591719) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:12:44 -- STEP: 15750/30063 -- GLOBAL_STEP: 15750[0m |
|
| > loss_text_ce: 0.04876994341611862 (0.056408973392512954) |
|
| > loss_mel_ce: 4.465559482574463 (4.775703950775983) |
|
| > loss: 1.1285823583602905 (1.208028231227208) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3366 (0.30719158611600295) |
|
| > loader_time: 0.0084 (0.009207584214588933) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:13:10 -- STEP: 15800/30063 -- GLOBAL_STEP: 15800[0m |
|
| > loss_text_ce: 0.04530955106019974 (0.05636346678332049) |
|
| > loss_mel_ce: 4.376796245574951 (4.774217996220035) |
|
| > loss: 1.1055264472961426 (1.2076453659428819) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2742 (0.3072188647185693) |
|
| > loader_time: 0.0091 (0.00920685861684097) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:13:37 -- STEP: 15850/30063 -- GLOBAL_STEP: 15850[0m |
|
| > loss_text_ce: 0.04494616389274597 (0.05631750497485174) |
|
| > loss_mel_ce: 4.261308670043945 (4.772812532491101) |
|
| > loss: 1.0765637159347534 (1.2072825095555753) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2524 (0.30724659160108525) |
|
| > loader_time: 0.0091 (0.00920619138783835) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:14:02 -- STEP: 15900/30063 -- GLOBAL_STEP: 15900[0m |
|
| > loss_text_ce: 0.0409160777926445 (0.05627056751657946) |
|
| > loss_mel_ce: 4.369264125823975 (4.771005953077989) |
|
| > loss: 1.1025450229644775 (1.2068191303284657) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2047 (0.3072245821412997) |
|
| > loader_time: 0.0092 (0.00920683652349986) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:14:27 -- STEP: 15950/30063 -- GLOBAL_STEP: 15950[0m |
|
| > loss_text_ce: 0.04240712523460388 (0.05622717815460072) |
|
| > loss_mel_ce: 4.5367255210876465 (4.769575938999082) |
|
| > loss: 1.1447831392288208 (1.2064507794940726) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3765 (0.30720732385461913) |
|
| > loader_time: 0.0093 (0.009206429932932102) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:14:52 -- STEP: 16000/30063 -- GLOBAL_STEP: 16000[0m |
|
| > loss_text_ce: 0.03799178823828697 (0.05618241453706276) |
|
| > loss_mel_ce: 3.8519790172576904 (4.767783997043956) |
|
| > loss: 0.9724926948547363 (1.2059916030764573) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2501 (0.30717216457426477) |
|
| > loader_time: 0.0089 (0.009206162214279156) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_16000.pth |
|
|
|
[1m --> TIME: 2024-10-26 10:15:29 -- STEP: 16050/30063 -- GLOBAL_STEP: 16050[0m |
|
| > loss_text_ce: 0.0433027409017086 (0.056138138589782775) |
|
| > loss_mel_ce: 3.5416133403778076 (4.7660342976163) |
|
| > loss: 0.8962290287017822 (1.2055431092268203) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2751 (0.3071707897245691) |
|
| > loader_time: 0.0092 (0.009206670674950887) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:15:55 -- STEP: 16100/30063 -- GLOBAL_STEP: 16100[0m |
|
| > loss_text_ce: 0.042961958795785904 (0.05609324514680757) |
|
| > loss_mel_ce: 4.808752059936523 (4.764452935790413) |
|
| > loss: 1.2129285335540771 (1.2051365454152498) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2564 (0.30716367959976126) |
|
| > loader_time: 0.0091 (0.009205960072345596) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:16:21 -- STEP: 16150/30063 -- GLOBAL_STEP: 16150[0m |
|
| > loss_text_ce: 0.038537364453077316 (0.05604989977513109) |
|
| > loss_mel_ce: 4.123438835144043 (4.763031597506509) |
|
| > loss: 1.0404940843582153 (1.2047703745158458) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2584 (0.30717197275014047) |
|
| > loader_time: 0.0088 (0.009205534686983166) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:16:46 -- STEP: 16200/30063 -- GLOBAL_STEP: 16200[0m |
|
| > loss_text_ce: 0.04190023988485336 (0.05600547933238145) |
|
| > loss_mel_ce: 4.167117595672607 (4.7617017441619955) |
|
| > loss: 1.0522544384002686 (1.204426806068714) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2715 (0.3071417335080505) |
|
| > loader_time: 0.0088 (0.009204869314476276) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:17:13 -- STEP: 16250/30063 -- GLOBAL_STEP: 16250[0m |
|
| > loss_text_ce: 0.04096363112330437 (0.055963666643316826) |
|
| > loss_mel_ce: 4.232492446899414 (4.760209155038676) |
|
| > loss: 1.0683640241622925 (1.2040432056096877) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2873 (0.3071574671085058) |
|
| > loader_time: 0.0095 (0.009204269013038028) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:17:39 -- STEP: 16300/30063 -- GLOBAL_STEP: 16300[0m |
|
| > loss_text_ce: 0.03716060519218445 (0.05591825027795829) |
|
| > loss_mel_ce: 3.8711676597595215 (4.758803826007364) |
|
| > loss: 0.9770820736885071 (1.2036805192714812) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2057 (0.30717195690775145) |
|
| > loader_time: 0.009 (0.009203778951445953) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:18:05 -- STEP: 16350/30063 -- GLOBAL_STEP: 16350[0m |
|
| > loss_text_ce: 0.03891333192586899 (0.05587286821677587) |
|
| > loss_mel_ce: 4.701501846313477 (4.75744706156785) |
|
| > loss: 1.1851037740707397 (1.2033299826409103) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.374 (0.3071844811979048) |
|
| > loader_time: 0.009 (0.009203296757619297) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:18:30 -- STEP: 16400/30063 -- GLOBAL_STEP: 16400[0m |
|
| > loss_text_ce: 0.04554561898112297 (0.0558287994188778) |
|
| > loss_mel_ce: 4.241091251373291 (4.75582186351461) |
|
| > loss: 1.0716592073440552 (1.2029126659544496) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2587 (0.30715752661228113) |
|
| > loader_time: 0.0089 (0.009202923236823644) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:18:56 -- STEP: 16450/30063 -- GLOBAL_STEP: 16450[0m |
|
| > loss_text_ce: 0.044093504548072815 (0.05578888438913769) |
|
| > loss_mel_ce: 4.322441101074219 (4.754467945171334) |
|
| > loss: 1.091633677482605 (1.2025642076198084) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2325 (0.3071497359174356) |
|
| > loader_time: 0.0086 (0.009202700632321417) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:19:22 -- STEP: 16500/30063 -- GLOBAL_STEP: 16500[0m |
|
| > loss_text_ce: 0.04357598349452019 (0.05574395423382511) |
|
| > loss_mel_ce: 3.8082895278930664 (4.752983321637808) |
|
| > loss: 0.9629663825035095 (1.2021818191716158) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2344 (0.3071573809132424) |
|
| > loader_time: 0.0092 (0.009202358144702315) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:19:48 -- STEP: 16550/30063 -- GLOBAL_STEP: 16550[0m |
|
| > loss_text_ce: 0.04050175100564957 (0.055700317655670226) |
|
| > loss_mel_ce: 4.3687872886657715 (4.751424171902973) |
|
| > loss: 1.1023222208023071 (1.2017811225836361) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2637 (0.30718870956732) |
|
| > loader_time: 0.0091 (0.009201943334135746) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:20:14 -- STEP: 16600/30063 -- GLOBAL_STEP: 16600[0m |
|
| > loss_text_ce: 0.04490629583597183 (0.055658726930079595) |
|
| > loss_mel_ce: 4.52753210067749 (4.750370768840044) |
|
| > loss: 1.1431095600128174 (1.201507374123636) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2327 (0.3071810895850853) |
|
| > loader_time: 0.009 (0.009201699251151928) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:20:40 -- STEP: 16650/30063 -- GLOBAL_STEP: 16650[0m |
|
| > loss_text_ce: 0.04312106594443321 (0.05561805011631839) |
|
| > loss_mel_ce: 4.438338756561279 (4.748976845483512) |
|
| > loss: 1.1203649044036865 (1.2011487240762677) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2471 (0.30717465711427455) |
|
| > loader_time: 0.0087 (0.009200727545821252) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:21:06 -- STEP: 16700/30063 -- GLOBAL_STEP: 16700[0m |
|
| > loss_text_ce: 0.04048642888665199 (0.0555751031224509) |
|
| > loss_mel_ce: 4.307858943939209 (4.7476146745538905) |
|
| > loss: 1.0870863199234009 (1.2007974446141072) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2548 (0.30717023701011) |
|
| > loader_time: 0.009 (0.009200155892058028) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:21:31 -- STEP: 16750/30063 -- GLOBAL_STEP: 16750[0m |
|
| > loss_text_ce: 0.040048204362392426 (0.055533742187183203) |
|
| > loss_mel_ce: 4.694337368011475 (4.745984354175729) |
|
| > loss: 1.1835963726043701 (1.2003795242736583) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4174 (0.30714778616890914) |
|
| > loader_time: 0.0111 (0.009199771980741112) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:21:57 -- STEP: 16800/30063 -- GLOBAL_STEP: 16800[0m |
|
| > loss_text_ce: 0.039602652192115784 (0.05548979050812436) |
|
| > loss_mel_ce: 4.281269550323486 (4.744260021774532) |
|
| > loss: 1.0802180767059326 (1.1999374532664102) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2064 (0.30714156932773984) |
|
| > loader_time: 0.0086 (0.009199361119951502) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:22:23 -- STEP: 16850/30063 -- GLOBAL_STEP: 16850[0m |
|
| > loss_text_ce: 0.046769265085458755 (0.05544806770228659) |
|
| > loss_mel_ce: 3.8877689838409424 (4.742518375068449) |
|
| > loss: 0.9836345911026001 (1.1994916108843123) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3291 (0.30716165212919816) |
|
| > loader_time: 0.0095 (0.009199207204031997) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:22:49 -- STEP: 16900/30063 -- GLOBAL_STEP: 16900[0m |
|
| > loss_text_ce: 0.03546395152807236 (0.055405777855693505) |
|
| > loss_mel_ce: 4.316732883453369 (4.741041592324261) |
|
| > loss: 1.0880491733551025 (1.199111842724698) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2015 (0.3071435513439962) |
|
| > loader_time: 0.0086 (0.009198224784354468) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:23:14 -- STEP: 16950/30063 -- GLOBAL_STEP: 16950[0m |
|
| > loss_text_ce: 0.039047811180353165 (0.05536647455772283) |
|
| > loss_mel_ce: 3.509335517883301 (4.73972156647031) |
|
| > loss: 0.8870958089828491 (1.198772010458605) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2995 (0.3071252404834661) |
|
| > loader_time: 0.0088 (0.009197952669981992) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:23:39 -- STEP: 17000/30063 -- GLOBAL_STEP: 17000[0m |
|
| > loss_text_ce: 0.03945689648389816 (0.05532348956540219) |
|
| > loss_mel_ce: 3.2146244049072266 (4.738134764713387) |
|
| > loss: 0.8135203123092651 (1.198364563777165) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3078 (0.3071135703255141) |
|
| > loader_time: 0.01 (0.009197773414499604) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:24:05 -- STEP: 17050/30063 -- GLOBAL_STEP: 17050[0m |
|
| > loss_text_ce: 0.044667892158031464 (0.05528254840140635) |
|
| > loss_mel_ce: 3.9544951915740967 (4.736727894506136) |
|
| > loss: 0.9997907876968384 (1.1980026109407365) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.503 (0.30712926900981036) |
|
| > loader_time: 0.0102 (0.009197452676610846) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:24:31 -- STEP: 17100/30063 -- GLOBAL_STEP: 17100[0m |
|
| > loss_text_ce: 0.047118328511714935 (0.055242234962986844) |
|
| > loss_mel_ce: 4.508616924285889 (4.735332914834819) |
|
| > loss: 1.138933777809143 (1.197643787655216) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.186 (0.3071018864118558) |
|
| > loader_time: 0.0099 (0.00919719725324395) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:24:57 -- STEP: 17150/30063 -- GLOBAL_STEP: 17150[0m |
|
| > loss_text_ce: 0.04781264066696167 (0.055202827463605234) |
|
| > loss_mel_ce: 3.8932547569274902 (4.7337684449927) |
|
| > loss: 0.9852668642997742 (1.1972428183027317) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2611 (0.3071117809840604) |
|
| > loader_time: 0.0088 (0.009196675233868731) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:25:24 -- STEP: 17200/30063 -- GLOBAL_STEP: 17200[0m |
|
| > loss_text_ce: 0.040523264557123184 (0.05516247089259144) |
|
| > loss_mel_ce: 4.670137882232666 (4.732437047625685) |
|
| > loss: 1.1776652336120605 (1.1968998798312132) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2723 (0.3071597906739207) |
|
| > loader_time: 0.0097 (0.009196376301521467) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:25:48 -- STEP: 17250/30063 -- GLOBAL_STEP: 17250[0m |
|
| > loss_text_ce: 0.035356100648641586 (0.05512224782312252) |
|
| > loss_mel_ce: 4.238561630249023 (4.730935570661562) |
|
| > loss: 1.0684794187545776 (1.1965144548312465) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2326 (0.30711116070678196) |
|
| > loader_time: 0.0087 (0.009196073393890813) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:26:14 -- STEP: 17300/30063 -- GLOBAL_STEP: 17300[0m |
|
| > loss_text_ce: 0.04317530617117882 (0.05508142309618642) |
|
| > loss_mel_ce: 3.9369611740112305 (4.729399691647866) |
|
| > loss: 0.9950340986251831 (1.196120278890421) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3566 (0.30709923429985186) |
|
| > loader_time: 0.01 (0.0091955502736086) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:26:40 -- STEP: 17350/30063 -- GLOBAL_STEP: 17350[0m |
|
| > loss_text_ce: 0.039038512855768204 (0.0550402999720844) |
|
| > loss_mel_ce: 3.181663990020752 (4.727942509513756) |
|
| > loss: 0.8051756024360657 (1.1957457025971796) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3064 (0.307100561908754) |
|
| > loader_time: 0.0101 (0.009195734004809455) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:27:06 -- STEP: 17400/30063 -- GLOBAL_STEP: 17400[0m |
|
| > loss_text_ce: 0.03947627544403076 (0.05500068341478184) |
|
| > loss_mel_ce: 4.37392520904541 (4.726390517196428) |
|
| > loss: 1.1033504009246826 (1.1953478003610127) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2433 (0.3070966035470186) |
|
| > loader_time: 0.0086 (0.009194830601242755) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:27:32 -- STEP: 17450/30063 -- GLOBAL_STEP: 17450[0m |
|
| > loss_text_ce: 0.041066497564315796 (0.05496199092926465) |
|
| > loss_mel_ce: 3.897260904312134 (4.724871151180862) |
|
| > loss: 0.9845818281173706 (1.1949582857252181) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3108 (0.30711090228618987) |
|
| > loader_time: 0.0094 (0.009194785011531956) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:27:58 -- STEP: 17500/30063 -- GLOBAL_STEP: 17500[0m |
|
| > loss_text_ce: 0.03808281943202019 (0.05492267328287864) |
|
| > loss_mel_ce: 4.35214376449585 (4.723537221949434) |
|
| > loss: 1.0975565910339355 (1.1946149740082865) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2512 (0.3071255578994743) |
|
| > loader_time: 0.009 (0.009194582584926045) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:28:24 -- STEP: 17550/30063 -- GLOBAL_STEP: 17550[0m |
|
| > loss_text_ce: 0.043960507959127426 (0.05488439960824448) |
|
| > loss_mel_ce: 4.0187859535217285 (4.722077009548824) |
|
| > loss: 1.0156866312026978 (1.1942403524893292) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3378 (0.3071235515724892) |
|
| > loader_time: 0.0091 (0.00919423203862289) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:28:50 -- STEP: 17600/30063 -- GLOBAL_STEP: 17600[0m |
|
| > loss_text_ce: 0.038897883147001266 (0.05484520955540398) |
|
| > loss_mel_ce: 4.040969371795654 (4.7205007775398755) |
|
| > loss: 1.0199668407440186 (1.193836496983739) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1888 (0.3071462061865755) |
|
| > loader_time: 0.0085 (0.009194205443967457) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:29:16 -- STEP: 17650/30063 -- GLOBAL_STEP: 17650[0m |
|
| > loss_text_ce: 0.04163587838411331 (0.05480678415733241) |
|
| > loss_mel_ce: 4.473209857940674 (4.719046802129009) |
|
| > loss: 1.128711462020874 (1.1934633967761605) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4451 (0.30713789226650756) |
|
| > loader_time: 0.0097 (0.009194080633752388) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:29:42 -- STEP: 17700/30063 -- GLOBAL_STEP: 17700[0m |
|
| > loss_text_ce: 0.039814453572034836 (0.054767702346678035) |
|
| > loss_mel_ce: 4.0652008056640625 (4.717437150774694) |
|
| > loss: 1.0262538194656372 (1.19305121347729) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3101 (0.3071467101910684) |
|
| > loader_time: 0.0096 (0.009194013385449406) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:30:09 -- STEP: 17750/30063 -- GLOBAL_STEP: 17750[0m |
|
| > loss_text_ce: 0.04518073424696922 (0.05473061914804943) |
|
| > loss_mel_ce: 4.781771659851074 (4.716332380979824) |
|
| > loss: 1.2067381143569946 (1.192765750233555) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3357 (0.30718487059902044) |
|
| > loader_time: 0.0082 (0.009193577752986402) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:30:35 -- STEP: 17800/30063 -- GLOBAL_STEP: 17800[0m |
|
| > loss_text_ce: 0.04087599366903305 (0.054692382112647576) |
|
| > loss_mel_ce: 3.6017773151397705 (4.714913690532182) |
|
| > loss: 0.9106633067131042 (1.1924015183428691) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2468 (0.30719181855073047) |
|
| > loader_time: 0.0087 (0.009193222549524175) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:31:01 -- STEP: 17850/30063 -- GLOBAL_STEP: 17850[0m |
|
| > loss_text_ce: 0.03765598312020302 (0.054652519798537434) |
|
| > loss_mel_ce: 3.914109706878662 (4.713439393631205) |
|
| > loss: 0.9879414439201355 (1.1920229785348846) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3452 (0.3071975786011417) |
|
| > loader_time: 0.0093 (0.009193015258853148) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:31:27 -- STEP: 17900/30063 -- GLOBAL_STEP: 17900[0m |
|
| > loss_text_ce: 0.037318501621484756 (0.05461473889128605) |
|
| > loss_mel_ce: 3.9009342193603516 (4.711951555286693) |
|
| > loss: 0.9845631718635559 (1.1916415737327908) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2371 (0.30722389139942535) |
|
| > loader_time: 0.0085 (0.009192481467177719) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:31:53 -- STEP: 17950/30063 -- GLOBAL_STEP: 17950[0m |
|
| > loss_text_ce: 0.04095972329378128 (0.05457684978116338) |
|
| > loss_mel_ce: 4.091490268707275 (4.710444831542637) |
|
| > loss: 1.0331125259399414 (1.1912554205393722) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.359 (0.30723013289459494) |
|
| > loader_time: 0.009 (0.009192028085501387) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:32:19 -- STEP: 18000/30063 -- GLOBAL_STEP: 18000[0m |
|
| > loss_text_ce: 0.03939161077141762 (0.054537837034919025) |
|
| > loss_mel_ce: 3.916841745376587 (4.709029821938925) |
|
| > loss: 0.9890583157539368 (1.1908919149471646) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2091 (0.3072247031794649) |
|
| > loader_time: 0.0089 (0.009191448714998012) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_18000.pth |
|
|
|
[1m --> TIME: 2024-10-26 10:32:55 -- STEP: 18050/30063 -- GLOBAL_STEP: 18050[0m |
|
| > loss_text_ce: 0.04627082869410515 (0.05450113017328705) |
|
| > loss_mel_ce: 4.558797836303711 (4.707421590205362) |
|
| > loss: 1.151267170906067 (1.190480680284076) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3123 (0.30722545897201103) |
|
| > loader_time: 0.0089 (0.009192263188454558) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:33:20 -- STEP: 18100/30063 -- GLOBAL_STEP: 18100[0m |
|
| > loss_text_ce: 0.045661311596632004 (0.0544629276150743) |
|
| > loss_mel_ce: 4.6822099685668945 (4.705614430864858) |
|
| > loss: 1.181967854499817 (1.1900193397873653) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4081 (0.3071791991739637) |
|
| > loader_time: 0.0095 (0.009191941793452285) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:33:46 -- STEP: 18150/30063 -- GLOBAL_STEP: 18150[0m |
|
| > loss_text_ce: 0.03763343766331673 (0.054426144896788195) |
|
| > loss_mel_ce: 3.88830828666687 (4.704350131234501) |
|
| > loss: 0.9814854264259338 (1.1896940691937403) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.312 (0.3071629131201536) |
|
| > loader_time: 0.0112 (0.009191382161185077) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:34:12 -- STEP: 18200/30063 -- GLOBAL_STEP: 18200[0m |
|
| > loss_text_ce: 0.04023650661110878 (0.054387974888700175) |
|
| > loss_mel_ce: 3.9460573196411133 (4.702855506323179) |
|
| > loss: 0.9965734481811523 (1.1893108704522388) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2722 (0.30719377386700897) |
|
| > loader_time: 0.0095 (0.009191266154194898) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:34:37 -- STEP: 18250/30063 -- GLOBAL_STEP: 18250[0m |
|
| > loss_text_ce: 0.04055166617035866 (0.05435099944043628) |
|
| > loss_mel_ce: 3.8243253231048584 (4.701484044362402) |
|
| > loss: 0.9662192463874817 (1.1889587611074284) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3358 (0.3071709747183803) |
|
| > loader_time: 0.0089 (0.009191069825054824) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:35:04 -- STEP: 18300/30063 -- GLOBAL_STEP: 18300[0m |
|
| > loss_text_ce: 0.03768610954284668 (0.054315838021902894) |
|
| > loss_mel_ce: 4.158621788024902 (4.700495174334982) |
|
| > loss: 1.049077033996582 (1.188702753235078) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2248 (0.30720130322409356) |
|
| > loader_time: 0.0091 (0.009190986390973682) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:35:31 -- STEP: 18350/30063 -- GLOBAL_STEP: 18350[0m |
|
| > loss_text_ce: 0.04277918115258217 (0.054279178021632825) |
|
| > loss_mel_ce: 4.227875232696533 (4.699156409531246) |
|
| > loss: 1.067663550376892 (1.1883588970259644) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3822 (0.30723853582907) |
|
| > loader_time: 0.0097 (0.009191028987354391) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:35:57 -- STEP: 18400/30063 -- GLOBAL_STEP: 18400[0m |
|
| > loss_text_ce: 0.038747839629650116 (0.05424266141596353) |
|
| > loss_mel_ce: 3.8206591606140137 (4.6976212487402) |
|
| > loss: 0.9648517370223999 (1.1879659776583924) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2017 (0.3072556453813666) |
|
| > loader_time: 0.0089 (0.00919165252343466) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:36:22 -- STEP: 18450/30063 -- GLOBAL_STEP: 18450[0m |
|
| > loss_text_ce: 0.03624747321009636 (0.05420499921208454) |
|
| > loss_mel_ce: 4.963409423828125 (4.696255014478979) |
|
| > loss: 1.2499141693115234 (1.187615003547045) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.336 (0.3072300121519289) |
|
| > loader_time: 0.0092 (0.009191361610805415) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:36:48 -- STEP: 18500/30063 -- GLOBAL_STEP: 18500[0m |
|
| > loss_text_ce: 0.04001897573471069 (0.05416773069166637) |
|
| > loss_mel_ce: 4.327743053436279 (4.694811897277814) |
|
| > loss: 1.0919405221939087 (1.1872449071278415) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2456 (0.3072065354940042) |
|
| > loader_time: 0.0098 (0.009191492183788381) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:37:16 -- STEP: 18550/30063 -- GLOBAL_STEP: 18550[0m |
|
| > loss_text_ce: 0.03907039016485214 (0.05413121601641963) |
|
| > loss_mel_ce: 4.789095878601074 (4.693688231851194) |
|
| > loss: 1.207041621208191 (1.1869548621106965) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4006 (0.3072780712981094) |
|
| > loader_time: 0.0095 (0.009191907384003573) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:37:42 -- STEP: 18600/30063 -- GLOBAL_STEP: 18600[0m |
|
| > loss_text_ce: 0.04286431148648262 (0.05409700048546621) |
|
| > loss_mel_ce: 4.108419418334961 (4.692285915561883) |
|
| > loss: 1.0378209352493286 (1.1865957291600502) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2043 (0.30730634908522175) |
|
| > loader_time: 0.0105 (0.009191869151207692) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:38:09 -- STEP: 18650/30063 -- GLOBAL_STEP: 18650[0m |
|
| > loss_text_ce: 0.04038739949464798 (0.05406224778266583) |
|
| > loss_mel_ce: 4.320259094238281 (4.690964511942906) |
|
| > loss: 1.090161681175232 (1.1862566900700704) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4698 (0.3073324466454743) |
|
| > loader_time: 0.01 (0.009191822596593431) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:38:34 -- STEP: 18700/30063 -- GLOBAL_STEP: 18700[0m |
|
| > loss_text_ce: 0.03680761903524399 (0.054026252737736806) |
|
| > loss_mel_ce: 2.8311071395874023 (4.689311041270967) |
|
| > loss: 0.7169786691665649 (1.185834323649098) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2176 (0.30730730599897926) |
|
| > loader_time: 0.0082 (0.00919123400978861) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:39:00 -- STEP: 18750/30063 -- GLOBAL_STEP: 18750[0m |
|
| > loss_text_ce: 0.03845355659723282 (0.05399259481549248) |
|
| > loss_mel_ce: 4.2195868492126465 (4.688146561749756) |
|
| > loss: 1.0645101070404053 (1.1855347892634058) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3585 (0.30733494853973237) |
|
| > loader_time: 0.0091 (0.009191167424519833) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:39:26 -- STEP: 18800/30063 -- GLOBAL_STEP: 18800[0m |
|
| > loss_text_ce: 0.043420884758234024 (0.05395625832193378) |
|
| > loss_mel_ce: 4.564781665802002 (4.686790459168698) |
|
| > loss: 1.1520506143569946 (1.185186679486897) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4159 (0.3073350629147049) |
|
| > loader_time: 0.0101 (0.009190775054566376) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:39:52 -- STEP: 18850/30063 -- GLOBAL_STEP: 18850[0m |
|
| > loss_text_ce: 0.04348982125520706 (0.05391964659924833) |
|
| > loss_mel_ce: 4.379343032836914 (4.68544303458943) |
|
| > loss: 1.1057082414627075 (1.184840670403496) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2547 (0.30731246959625425) |
|
| > loader_time: 0.0089 (0.009190998962766582) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:40:19 -- STEP: 18900/30063 -- GLOBAL_STEP: 18900[0m |
|
| > loss_text_ce: 0.04539928957819939 (0.053884341423670636) |
|
| > loss_mel_ce: 4.425279140472412 (4.684283669953606) |
|
| > loss: 1.1176695823669434 (1.1845420029459781) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3204 (0.30736202522560263) |
|
| > loader_time: 0.0091 (0.00919123018860183) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:40:45 -- STEP: 18950/30063 -- GLOBAL_STEP: 18950[0m |
|
| > loss_text_ce: 0.041365984827280045 (0.053849544621473094) |
|
| > loss_mel_ce: 4.395998477935791 (4.682991876891527) |
|
| > loss: 1.1093411445617676 (1.1842103554705494) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.309 (0.3073627997954461) |
|
| > loader_time: 0.0093 (0.009191405678801896) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:41:11 -- STEP: 19000/30063 -- GLOBAL_STEP: 19000[0m |
|
| > loss_text_ce: 0.039157744497060776 (0.0538127648673558) |
|
| > loss_mel_ce: 4.922301769256592 (4.681818575030859) |
|
| > loss: 1.240364909172058 (1.1839078350851386) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2482 (0.3073559956048648) |
|
| > loader_time: 0.0086 (0.009191350623180963) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:41:36 -- STEP: 19050/30063 -- GLOBAL_STEP: 19050[0m |
|
| > loss_text_ce: 0.04159564897418022 (0.05378115444514992) |
|
| > loss_mel_ce: 4.3953070640563965 (4.680683386294529) |
|
| > loss: 1.1092256307601929 (1.1836161352611876) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2742 (0.30734349662550337) |
|
| > loader_time: 0.0087 (0.009191053095139205) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:42:01 -- STEP: 19100/30063 -- GLOBAL_STEP: 19100[0m |
|
| > loss_text_ce: 0.039685122668743134 (0.05374634405432244) |
|
| > loss_mel_ce: 4.304113864898682 (4.679344863429723) |
|
| > loss: 1.0859497785568237 (1.1832728019434724) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3324 (0.3073229928790573) |
|
| > loader_time: 0.0096 (0.009190451636988428) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:42:28 -- STEP: 19150/30063 -- GLOBAL_STEP: 19150[0m |
|
| > loss_text_ce: 0.03240034356713295 (0.05371191226237732) |
|
| > loss_mel_ce: 3.9657344818115234 (4.677964345294544) |
|
| > loss: 0.9995337128639221 (1.1829190644489544) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2881 (0.3073424306675275) |
|
| > loader_time: 0.0089 (0.00918986044104351) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:42:54 -- STEP: 19200/30063 -- GLOBAL_STEP: 19200[0m |
|
| > loss_text_ce: 0.04475318640470505 (0.05367737323103923) |
|
| > loss_mel_ce: 4.296286582946777 (4.676690604661883) |
|
| > loss: 1.0852599143981934 (1.1825919945320718) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2713 (0.3073449017604182) |
|
| > loader_time: 0.0086 (0.00918955376992619) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:43:19 -- STEP: 19250/30063 -- GLOBAL_STEP: 19250[0m |
|
| > loss_text_ce: 0.04122244194149971 (0.053643118428719355) |
|
| > loss_mel_ce: 3.654003858566284 (4.675108015766375) |
|
| > loss: 0.9238065481185913 (1.182187783606639) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.339 (0.30733390674343264) |
|
| > loader_time: 0.0087 (0.009189260780037182) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:43:44 -- STEP: 19300/30063 -- GLOBAL_STEP: 19300[0m |
|
| > loss_text_ce: 0.04265483468770981 (0.05360711608680403) |
|
| > loss_mel_ce: 3.998814344406128 (4.673961322097562) |
|
| > loss: 1.0103672742843628 (1.181892109602843) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2099 (0.3073148081710292) |
|
| > loader_time: 0.0086 (0.009188758029838873) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:44:11 -- STEP: 19350/30063 -- GLOBAL_STEP: 19350[0m |
|
| > loss_text_ce: 0.04123488813638687 (0.0535740486001089) |
|
| > loss_mel_ce: 4.072597503662109 (4.672655287281778) |
|
| > loss: 1.0284581184387207 (1.1815573340330925) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2621 (0.3073308386913555) |
|
| > loader_time: 0.0084 (0.009188578541580663) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:44:37 -- STEP: 19400/30063 -- GLOBAL_STEP: 19400[0m |
|
| > loss_text_ce: 0.04022153839468956 (0.05354294094132225) |
|
| > loss_mel_ce: 4.594501972198486 (4.671623129721751) |
|
| > loss: 1.1586809158325195 (1.1812915177228511) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2049 (0.30735428056765973) |
|
| > loader_time: 0.0098 (0.009188633481251773) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:45:02 -- STEP: 19450/30063 -- GLOBAL_STEP: 19450[0m |
|
| > loss_text_ce: 0.03855489194393158 (0.05351259909206292) |
|
| > loss_mel_ce: 3.98496413230896 (4.670279842783656) |
|
| > loss: 1.0058797597885132 (1.1809481105277335) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2673 (0.3073229581960358) |
|
| > loader_time: 0.0091 (0.009188325018993018) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:45:28 -- STEP: 19500/30063 -- GLOBAL_STEP: 19500[0m |
|
| > loss_text_ce: 0.04035898670554161 (0.05347550082244919) |
|
| > loss_mel_ce: 4.22078800201416 (4.668999898653742) |
|
| > loss: 1.0652867555618286 (1.1806188499346748) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3737 (0.3073336713130645) |
|
| > loader_time: 0.0094 (0.009187824053642019) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:45:54 -- STEP: 19550/30063 -- GLOBAL_STEP: 19550[0m |
|
| > loss_text_ce: 0.04432866349816322 (0.05344095224900456) |
|
| > loss_mel_ce: 4.526479244232178 (4.667457677885067) |
|
| > loss: 1.1427019834518433 (1.180224657598357) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3362 (0.30734027141805087) |
|
| > loader_time: 0.0091 (0.00918779335363436) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:46:19 -- STEP: 19600/30063 -- GLOBAL_STEP: 19600[0m |
|
| > loss_text_ce: 0.04100652039051056 (0.05340917773715832) |
|
| > loss_mel_ce: 4.7256927490234375 (4.666116580440052) |
|
| > loss: 1.191674828529358 (1.17988143960736) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2052 (0.3073100183083074) |
|
| > loader_time: 0.0087 (0.009187831562392506) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:46:44 -- STEP: 19650/30063 -- GLOBAL_STEP: 19650[0m |
|
| > loss_text_ce: 0.04094251990318298 (0.05337666634726145) |
|
| > loss_mel_ce: 4.526086330413818 (4.664628548294511) |
|
| > loss: 1.1417572498321533 (1.1795013037225344) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3789 (0.30727542542319125) |
|
| > loader_time: 0.0092 (0.009188044768859965) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:47:08 -- STEP: 19700/30063 -- GLOBAL_STEP: 19700[0m |
|
| > loss_text_ce: 0.03891918435692787 (0.05334232825237467) |
|
| > loss_mel_ce: 4.481374740600586 (4.6631915526825525) |
|
| > loss: 1.1300734281539917 (1.1791334702883862) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2082 (0.3072207989305395) |
|
| > loader_time: 0.009 (0.009188329640983867) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:47:33 -- STEP: 19750/30063 -- GLOBAL_STEP: 19750[0m |
|
| > loss_text_ce: 0.04698820039629936 (0.05331096855541558) |
|
| > loss_mel_ce: 4.23529577255249 (4.66202698322487) |
|
| > loss: 1.070570945739746 (1.1788344879904855) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2584 (0.3071934211465363) |
|
| > loader_time: 0.0086 (0.009188182239290965) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:47:59 -- STEP: 19800/30063 -- GLOBAL_STEP: 19800[0m |
|
| > loss_text_ce: 0.04405956715345383 (0.0532794013753005) |
|
| > loss_mel_ce: 4.631063938140869 (4.660861477611016) |
|
| > loss: 1.1687809228897095 (1.1785352197951708) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1853 (0.3071987923347575) |
|
| > loader_time: 0.0103 (0.00918792109296773) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:48:25 -- STEP: 19850/30063 -- GLOBAL_STEP: 19850[0m |
|
| > loss_text_ce: 0.03851671516895294 (0.053249370391745564) |
|
| > loss_mel_ce: 3.5309698581695557 (4.659528870330325) |
|
| > loss: 0.892371654510498 (1.1781945602209163) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3047 (0.3071965130750704) |
|
| > loader_time: 0.0098 (0.009187988790516827) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:48:50 -- STEP: 19900/30063 -- GLOBAL_STEP: 19900[0m |
|
| > loss_text_ce: 0.04171811789274216 (0.05321625638603401) |
|
| > loss_mel_ce: 4.018099308013916 (4.658145580687092) |
|
| > loss: 1.0149543285369873 (1.177840459301839) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2089 (0.3071768002773633) |
|
| > loader_time: 0.0087 (0.009188004738122315) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:49:16 -- STEP: 19950/30063 -- GLOBAL_STEP: 19950[0m |
|
| > loss_text_ce: 0.04186498373746872 (0.05318623755734054) |
|
| > loss_mel_ce: 4.34715461730957 (4.65699904649775) |
|
| > loss: 1.0972548723220825 (1.1775463210401027) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.262 (0.30717756547425756) |
|
| > loader_time: 0.0097 (0.009187532463169272) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:49:43 -- STEP: 20000/30063 -- GLOBAL_STEP: 20000[0m |
|
| > loss_text_ce: 0.03966930881142616 (0.05315565361399189) |
|
| > loss_mel_ce: 4.5417327880859375 (4.655702110862709) |
|
| > loss: 1.1453505754470825 (1.1772144411295595) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3017 (0.30719177695512606) |
|
| > loader_time: 0.0146 (0.00918776695728295) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_20000.pth |
|
|
|
[1m --> TIME: 2024-10-26 10:50:18 -- STEP: 20050/30063 -- GLOBAL_STEP: 20050[0m |
|
| > loss_text_ce: 0.040113337337970734 (0.05312249194338271) |
|
| > loss_mel_ce: 4.635109901428223 (4.654253286697002) |
|
| > loss: 1.1688058376312256 (1.176843944675007) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2776 (0.30716500724640106) |
|
| > loader_time: 0.0086 (0.009189970368458962) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:50:43 -- STEP: 20100/30063 -- GLOBAL_STEP: 20100[0m |
|
| > loss_text_ce: 0.04406638443470001 (0.05309109311884449) |
|
| > loss_mel_ce: 4.593681812286377 (4.652959534599984) |
|
| > loss: 1.1594370603561401 (1.1765126569443027) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2604 (0.3071218568531414) |
|
| > loader_time: 0.0091 (0.009190842286864252) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:51:09 -- STEP: 20150/30063 -- GLOBAL_STEP: 20150[0m |
|
| > loss_text_ce: 0.040512971580028534 (0.05305906565365351) |
|
| > loss_mel_ce: 4.103840351104736 (4.65190357727683) |
|
| > loss: 1.0360883474349976 (1.1762406607301252) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2997 (0.307124453466522) |
|
| > loader_time: 0.0092 (0.009190465522463139) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:51:35 -- STEP: 20200/30063 -- GLOBAL_STEP: 20200[0m |
|
| > loss_text_ce: 0.03967973217368126 (0.05302752420177097) |
|
| > loss_mel_ce: 3.8816778659820557 (4.650689372827489) |
|
| > loss: 0.9803394079208374 (1.1759292242462043) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1878 (0.3071279587485978) |
|
| > loader_time: 0.0086 (0.009190058779008244) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:51:59 -- STEP: 20250/30063 -- GLOBAL_STEP: 20250[0m |
|
| > loss_text_ce: 0.03907120227813721 (0.05299581385689003) |
|
| > loss_mel_ce: 3.5745115280151367 (4.649448014023842) |
|
| > loss: 0.9033956527709961 (1.1756109569514188) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3371 (0.30707858004393157) |
|
| > loader_time: 0.0099 (0.009189494780552166) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:52:26 -- STEP: 20300/30063 -- GLOBAL_STEP: 20300[0m |
|
| > loss_text_ce: 0.036256466060876846 (0.05296649813762131) |
|
| > loss_mel_ce: 3.6050047874450684 (4.648316768965677) |
|
| > loss: 0.9103153347969055 (1.1753208167623361) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2463 (0.3071224105534272) |
|
| > loader_time: 0.0089 (0.009189334773077677) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:52:50 -- STEP: 20350/30063 -- GLOBAL_STEP: 20350[0m |
|
| > loss_text_ce: 0.040249381214380264 (0.05293420022367245) |
|
| > loss_mel_ce: 3.2418136596679688 (4.6468234605578065) |
|
| > loss: 0.8205157518386841 (1.174939415185393) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.285 (0.3070755416813856) |
|
| > loader_time: 0.009 (0.009189981312950863) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:53:16 -- STEP: 20400/30063 -- GLOBAL_STEP: 20400[0m |
|
| > loss_text_ce: 0.03565439209342003 (0.05290121401185338) |
|
| > loss_mel_ce: 3.0450804233551025 (4.64562562886404) |
|
| > loss: 0.7701836824417114 (1.17463171071281) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2456 (0.3070786636717152) |
|
| > loader_time: 0.0091 (0.00919034364176724) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:53:42 -- STEP: 20450/30063 -- GLOBAL_STEP: 20450[0m |
|
| > loss_text_ce: 0.035617001354694366 (0.05286889268936971) |
|
| > loss_mel_ce: 3.33392333984375 (4.644274364392132) |
|
| > loss: 0.8423851132392883 (1.1742858142753612) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.257 (0.3070792848731576) |
|
| > loader_time: 0.01 (0.00919097957517807) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:54:09 -- STEP: 20500/30063 -- GLOBAL_STEP: 20500[0m |
|
| > loss_text_ce: 0.03900231793522835 (0.052836796043395126) |
|
| > loss_mel_ce: 3.4845218658447266 (4.642847164642495) |
|
| > loss: 0.8808810710906982 (1.1739209901995706) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2713 (0.30709759148155497) |
|
| > loader_time: 0.0091 (0.009191057449433783) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:54:34 -- STEP: 20550/30063 -- GLOBAL_STEP: 20550[0m |
|
| > loss_text_ce: 0.0343870185315609 (0.052806691399836486) |
|
| > loss_mel_ce: 4.404526710510254 (4.641459193392076) |
|
| > loss: 1.109728455543518 (1.1735664712222478) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3833 (0.30706847208259486) |
|
| > loader_time: 0.0093 (0.009191071157617792) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:55:01 -- STEP: 20600/30063 -- GLOBAL_STEP: 20600[0m |
|
| > loss_text_ce: 0.038114283233881 (0.052776485104927454) |
|
| > loss_mel_ce: 3.670222759246826 (4.639957707599505) |
|
| > loss: 0.9270842671394348 (1.1731835482074155) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2059 (0.3071160960081686) |
|
| > loader_time: 0.0085 (0.009190995855238752) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:55:26 -- STEP: 20650/30063 -- GLOBAL_STEP: 20650[0m |
|
| > loss_text_ce: 0.03908388689160347 (0.05274393912025918) |
|
| > loss_mel_ce: 3.702895402908325 (4.638507560829248) |
|
| > loss: 0.9354948401451111 (1.1728128750070175) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.297 (0.3070936328620053) |
|
| > loader_time: 0.0091 (0.009191087106238185) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:55:53 -- STEP: 20700/30063 -- GLOBAL_STEP: 20700[0m |
|
| > loss_text_ce: 0.03758860379457474 (0.05271460248787248) |
|
| > loss_mel_ce: 3.820145845413208 (4.6373168854782305) |
|
| > loss: 0.9644336104393005 (1.1725078720084632) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2233 (0.307122787507833) |
|
| > loader_time: 0.009 (0.009190786771728146) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:56:19 -- STEP: 20750/30063 -- GLOBAL_STEP: 20750[0m |
|
| > loss_text_ce: 0.04117259755730629 (0.052685326438082845) |
|
| > loss_mel_ce: 4.608562469482422 (4.6361938878599185) |
|
| > loss: 1.1624337434768677 (1.1722198035803113) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3354 (0.30714138657213425) |
|
| > loader_time: 0.0092 (0.009190648458090101) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:56:47 -- STEP: 20800/30063 -- GLOBAL_STEP: 20800[0m |
|
| > loss_text_ce: 0.03850547596812248 (0.0526558929675508) |
|
| > loss_mel_ce: 4.173282623291016 (4.635117725936241) |
|
| > loss: 1.0529470443725586 (1.1719434047289687) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2452 (0.30721105529711545) |
|
| > loader_time: 0.0091 (0.009190512654872973) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:57:13 -- STEP: 20850/30063 -- GLOBAL_STEP: 20850[0m |
|
| > loss_text_ce: 0.04056980088353157 (0.052624620328650486) |
|
| > loss_mel_ce: 4.125818729400635 (4.633705015148165) |
|
| > loss: 1.0415971279144287 (1.171582408866143) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3027 (0.3072311692843881) |
|
| > loader_time: 0.009 (0.009190681901195288) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:57:38 -- STEP: 20900/30063 -- GLOBAL_STEP: 20900[0m |
|
| > loss_text_ce: 0.0405069924890995 (0.05259558901787859) |
|
| > loss_mel_ce: 3.822497844696045 (4.632570616411795) |
|
| > loss: 0.9657512307167053 (1.1712915513532505) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.1748 (0.3072004677576288) |
|
| > loader_time: 0.01 (0.009190755757418482) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:58:04 -- STEP: 20950/30063 -- GLOBAL_STEP: 20950[0m |
|
| > loss_text_ce: 0.04597364366054535 (0.052563360507971064) |
|
| > loss_mel_ce: 4.122974395751953 (4.631258367479246) |
|
| > loss: 1.0422370433807373 (1.1709554320047393) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2611 (0.30720688683320674) |
|
| > loader_time: 0.0084 (0.00919065575611044) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:58:31 -- STEP: 21000/30063 -- GLOBAL_STEP: 21000[0m |
|
| > loss_text_ce: 0.03943745419383049 (0.05253277189116024) |
|
| > loss_mel_ce: 3.9351677894592285 (4.629883818671788) |
|
| > loss: 0.9936513304710388 (1.1706041476442632) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.173 (0.30722976586932005) |
|
| > loader_time: 0.0088 (0.009190231800079274) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:58:58 -- STEP: 21050/30063 -- GLOBAL_STEP: 21050[0m |
|
| > loss_text_ce: 0.03727870434522629 (0.052501528492360956) |
|
| > loss_mel_ce: 4.0858025550842285 (4.6286161357036715) |
|
| > loss: 1.0307703018188477 (1.1702794160525694) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2515 (0.3072473909849205) |
|
| > loader_time: 0.008 (0.00918997100866315) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:59:23 -- STEP: 21100/30063 -- GLOBAL_STEP: 21100[0m |
|
| > loss_text_ce: 0.03987004607915878 (0.05247098067281003) |
|
| > loss_mel_ce: 3.704885244369507 (4.627184383349368) |
|
| > loss: 0.936188817024231 (1.169913840994444) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3947 (0.3072375922406435) |
|
| > loader_time: 0.0085 (0.009189486503601005) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 10:59:47 -- STEP: 21150/30063 -- GLOBAL_STEP: 21150[0m |
|
| > loss_text_ce: 0.04270370304584503 (0.05244056863644219) |
|
| > loss_mel_ce: 3.9557247161865234 (4.625894223071132) |
|
| > loss: 0.9996070861816406 (1.1695836979192975) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3983 (0.307186076883445) |
|
| > loader_time: 0.0095 (0.009188881867320763) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:00:14 -- STEP: 21200/30063 -- GLOBAL_STEP: 21200[0m |
|
| > loss_text_ce: 0.03610064461827278 (0.05241090966221634) |
|
| > loss_mel_ce: 4.2351274490356445 (4.624847077828509) |
|
| > loss: 1.0678070783615112 (1.1693144968911087) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.241 (0.30719188773406936) |
|
| > loader_time: 0.0085 (0.009188843403222362) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:00:39 -- STEP: 21250/30063 -- GLOBAL_STEP: 21250[0m |
|
| > loss_text_ce: 0.041630182415246964 (0.052382596805428616) |
|
| > loss_mel_ce: 4.126216888427734 (4.623752812340658) |
|
| > loss: 1.0419617891311646 (1.1690338523079296) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2874 (0.30717710018157773) |
|
| > loader_time: 0.0093 (0.009188362671347159) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:01:05 -- STEP: 21300/30063 -- GLOBAL_STEP: 21300[0m |
|
| > loss_text_ce: 0.03775344416499138 (0.05235207514679964) |
|
| > loss_mel_ce: 4.480717182159424 (4.622388183466126) |
|
| > loss: 1.129617691040039 (1.1686850646599845) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2037 (0.3071649416511589) |
|
| > loader_time: 0.0082 (0.009187705147434224) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:01:30 -- STEP: 21350/30063 -- GLOBAL_STEP: 21350[0m |
|
| > loss_text_ce: 0.038119133561849594 (0.05232468417180422) |
|
| > loss_mel_ce: 4.122433662414551 (4.621168972814932) |
|
| > loss: 1.0401382446289062 (1.1683734142501048) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3129 (0.3071625292329073) |
|
| > loader_time: 0.0092 (0.009187306174070483) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:01:56 -- STEP: 21400/30063 -- GLOBAL_STEP: 21400[0m |
|
| > loss_text_ce: 0.04329284280538559 (0.05229686196345091) |
|
| > loss_mel_ce: 4.662398815155029 (4.62001980051814) |
|
| > loss: 1.1764229536056519 (1.1680791656324725) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.345 (0.30714895329742575) |
|
| > loader_time: 0.0091 (0.009186642938685133) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:02:22 -- STEP: 21450/30063 -- GLOBAL_STEP: 21450[0m |
|
| > loss_text_ce: 0.03453602269291878 (0.052266238276479686) |
|
| > loss_mel_ce: 3.2324135303497314 (4.618913396821967) |
|
| > loss: 0.8167374134063721 (1.1677949087908692) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3785 (0.3071763623066415) |
|
| > loader_time: 0.0087 (0.00918596903483067) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:02:49 -- STEP: 21500/30063 -- GLOBAL_STEP: 21500[0m |
|
| > loss_text_ce: 0.03107278048992157 (0.05223769721007611) |
|
| > loss_mel_ce: 3.9824187755584717 (4.617637216867379) |
|
| > loss: 1.0033729076385498 (1.1674687285450924) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2466 (0.3071912837361157) |
|
| > loader_time: 0.0097 (0.009185925672220574) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:03:16 -- STEP: 21550/30063 -- GLOBAL_STEP: 21550[0m |
|
| > loss_text_ce: 0.04106273129582405 (0.05220937618488765) |
|
| > loss_mel_ce: 4.660557746887207 (4.616503187708382) |
|
| > loss: 1.1754051446914673 (1.1671781409920707) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3069 (0.30725190902807087) |
|
| > loader_time: 0.0092 (0.00918572351716782) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:03:43 -- STEP: 21600/30063 -- GLOBAL_STEP: 21600[0m |
|
| > loss_text_ce: 0.039408452808856964 (0.05218177615285458) |
|
| > loss_mel_ce: 4.199373245239258 (4.615542037001345) |
|
| > loss: 1.0596954822540283 (1.1669309532918244) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2334 (0.30726512268737277) |
|
| > loader_time: 0.0087 (0.009185421577206303) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:04:07 -- STEP: 21650/30063 -- GLOBAL_STEP: 21650[0m |
|
| > loss_text_ce: 0.04326842725276947 (0.05215292413845886) |
|
| > loss_mel_ce: 4.421598434448242 (4.614365937032526) |
|
| > loss: 1.1162166595458984 (1.1666297153127057) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2949 (0.30723910588314884) |
|
| > loader_time: 0.0101 (0.009184781012854389) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:04:33 -- STEP: 21700/30063 -- GLOBAL_STEP: 21700[0m |
|
| > loss_text_ce: 0.04013550281524658 (0.052125823927983025) |
|
| > loss_mel_ce: 4.48711633682251 (4.613531345727788) |
|
| > loss: 1.1318129301071167 (1.1664142924409804) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2728 (0.3072284306249286) |
|
| > loader_time: 0.0087 (0.009184870049700694) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:04:59 -- STEP: 21750/30063 -- GLOBAL_STEP: 21750[0m |
|
| > loss_text_ce: 0.04418960586190224 (0.05209927699072594) |
|
| > loss_mel_ce: 4.992228984832764 (4.612618956445258) |
|
| > loss: 1.259104609489441 (1.166179558381262) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.5141 (0.3072325243346968) |
|
| > loader_time: 0.0092 (0.009184153556823666) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:05:26 -- STEP: 21800/30063 -- GLOBAL_STEP: 21800[0m |
|
| > loss_text_ce: 0.038546960800886154 (0.05207121206943035) |
|
| > loss_mel_ce: 4.116634368896484 (4.611553029521867) |
|
| > loss: 1.0387953519821167 (1.1659060604282423) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2096 (0.3072610566156696) |
|
| > loader_time: 0.0096 (0.009184523077186021) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:05:51 -- STEP: 21850/30063 -- GLOBAL_STEP: 21850[0m |
|
| > loss_text_ce: 0.03858276829123497 (0.05204223401128414) |
|
| > loss_mel_ce: 4.0600666999816895 (4.610185740937881) |
|
| > loss: 1.0246623754501343 (1.1655569937654657) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3738 (0.3072579546651226) |
|
| > loader_time: 0.0086 (0.009184276441141922) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:06:17 -- STEP: 21900/30063 -- GLOBAL_STEP: 21900[0m |
|
| > loss_text_ce: 0.037075821310281754 (0.0520133289704991) |
|
| > loss_mel_ce: 4.262927055358887 (4.608931877819893) |
|
| > loss: 1.0750007629394531 (1.1652363017106113) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.202 (0.30725410943706016) |
|
| > loader_time: 0.0088 (0.009183614253997734) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:06:44 -- STEP: 21950/30063 -- GLOBAL_STEP: 21950[0m |
|
| > loss_text_ce: 0.035398125648498535 (0.0519866423378432) |
|
| > loss_mel_ce: 2.906306028366089 (4.60793621729064) |
|
| > loss: 0.7354260683059692 (1.1649807149266813) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3849 (0.3073070660593271) |
|
| > loader_time: 0.0099 (0.00918322452380058) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:07:11 -- STEP: 22000/30063 -- GLOBAL_STEP: 22000[0m |
|
| > loss_text_ce: 0.03972029685974121 (0.0519575720585204) |
|
| > loss_mel_ce: 4.178533554077148 (4.6066714007420675) |
|
| > loss: 1.0545635223388672 (1.1646572432247009) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.243 (0.30732843394712683) |
|
| > loader_time: 0.0083 (0.009182742649858584) |
|
|
|
|
|
> CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_22000.pth |
|
|
|
[1m --> TIME: 2024-10-26 11:07:48 -- STEP: 22050/30063 -- GLOBAL_STEP: 22050[0m |
|
| > loss_text_ce: 0.04076101630926132 (0.0519298816769638) |
|
| > loss_mel_ce: 4.162412643432617 (4.6057620146917575) |
|
| > loss: 1.0507934093475342 (1.1644229741053185) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2785 (0.30733182931973263) |
|
| > loader_time: 0.0095 (0.0091831833975655) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:08:12 -- STEP: 22100/30063 -- GLOBAL_STEP: 22100[0m |
|
| > loss_text_ce: 0.039491601288318634 (0.0519006260705276) |
|
| > loss_mel_ce: 4.1514716148376465 (4.6041506548272695) |
|
| > loss: 1.0477408170700073 (1.1640128202405933) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2075 (0.30729370191625593) |
|
| > loader_time: 0.0084 (0.009182746378005298) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:08:38 -- STEP: 22150/30063 -- GLOBAL_STEP: 22150[0m |
|
| > loss_text_ce: 0.04339059069752693 (0.051872100664416096) |
|
| > loss_mel_ce: 4.635831832885742 (4.602961483410697) |
|
| > loss: 1.169805645942688 (1.1637083960440515) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2337 (0.3073072986839556) |
|
| > loader_time: 0.0086 (0.009182630661647897) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:09:05 -- STEP: 22200/30063 -- GLOBAL_STEP: 22200[0m |
|
| > loss_text_ce: 0.0389290377497673 (0.05184510322019176) |
|
| > loss_mel_ce: 4.303701877593994 (4.601942484593568) |
|
| > loss: 1.0856577157974243 (1.1634468969810057) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2436 (0.30732055146414655) |
|
| > loader_time: 0.0089 (0.00918244513305451) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:09:31 -- STEP: 22250/30063 -- GLOBAL_STEP: 22250[0m |
|
| > loss_text_ce: 0.035264261066913605 (0.05181751855822748) |
|
| > loss_mel_ce: 3.979304790496826 (4.600691662981231) |
|
| > loss: 1.0036423206329346 (1.1631272954190655) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2619 (0.3073471188063002) |
|
| > loader_time: 0.0094 (0.009182179043801914) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:09:58 -- STEP: 22300/30063 -- GLOBAL_STEP: 22300[0m |
|
| > loss_text_ce: 0.03839513286948204 (0.05178987414429928) |
|
| > loss_mel_ce: 3.9766597747802734 (4.599667516738274) |
|
| > loss: 1.0037636756896973 (1.1628643477470826) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2758 (0.30737299542790264) |
|
| > loader_time: 0.0093 (0.009182253525396016) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:10:23 -- STEP: 22350/30063 -- GLOBAL_STEP: 22350[0m |
|
| > loss_text_ce: 0.039192430675029755 (0.051761807750001984) |
|
| > loss_mel_ce: 3.331636667251587 (4.598222293181673) |
|
| > loss: 0.8427072763442993 (1.1624960252555903) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2668 (0.3073613673941904) |
|
| > loader_time: 0.0088 (0.009181876705263464) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:10:49 -- STEP: 22400/30063 -- GLOBAL_STEP: 22400[0m |
|
| > loss_text_ce: 0.03900080546736717 (0.05173627272747178) |
|
| > loss_mel_ce: 4.584355354309082 (4.597258319620547) |
|
| > loss: 1.1558390855789185 (1.1622486481203507) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2332 (0.30735242268868557) |
|
| > loader_time: 0.0086 (0.009181692813123891) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:11:17 -- STEP: 22450/30063 -- GLOBAL_STEP: 22450[0m |
|
| > loss_text_ce: 0.038555581122636795 (0.051710038814685264) |
|
| > loss_mel_ce: 3.456609010696411 (4.596006860074563) |
|
| > loss: 0.8737911581993103 (1.161929224748117) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3444 (0.30740114551875264) |
|
| > loader_time: 0.0087 (0.009181055141185512) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:11:43 -- STEP: 22500/30063 -- GLOBAL_STEP: 22500[0m |
|
| > loss_text_ce: 0.039642881602048874 (0.05168120483424915) |
|
| > loss_mel_ce: 3.859917163848877 (4.59486852230493) |
|
| > loss: 0.9748899936676025 (1.161637431814929) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2885 (0.3074338149282642) |
|
| > loader_time: 0.0088 (0.009180644787682345) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:12:10 -- STEP: 22550/30063 -- GLOBAL_STEP: 22550[0m |
|
| > loss_text_ce: 0.0403783917427063 (0.051656063996139624) |
|
| > loss_mel_ce: 4.2151947021484375 (4.59392549969402) |
|
| > loss: 1.0638933181762695 (1.161395390953557) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2314 (0.3074529030793496) |
|
| > loader_time: 0.0089 (0.009180366025531295) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:12:36 -- STEP: 22600/30063 -- GLOBAL_STEP: 22600[0m |
|
| > loss_text_ce: 0.03614070266485214 (0.05162834915818749) |
|
| > loss_mel_ce: 3.447444200515747 (4.592749804490412) |
|
| > loss: 0.8708962202072144 (1.1610945384323004) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2497 (0.3074528685818704) |
|
| > loader_time: 0.0088 (0.009180006812104005) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:13:02 -- STEP: 22650/30063 -- GLOBAL_STEP: 22650[0m |
|
| > loss_text_ce: 0.04471220448613167 (0.05160094608994759) |
|
| > loss_mel_ce: 3.731722593307495 (4.591249055062414) |
|
| > loss: 0.9441087245941162 (1.1607125003079906) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.4452 (0.3074530590869968) |
|
| > loader_time: 0.0114 (0.00917980549877571) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:13:27 -- STEP: 22700/30063 -- GLOBAL_STEP: 22700[0m |
|
| > loss_text_ce: 0.03793908655643463 (0.05157473423046412) |
|
| > loss_mel_ce: 4.074444770812988 (4.59026586944309) |
|
| > loss: 1.0280959606170654 (1.1604601509245414) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.229 (0.30744104353866614) |
|
| > loader_time: 0.0087 (0.009179243453273603) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:13:53 -- STEP: 22750/30063 -- GLOBAL_STEP: 22750[0m |
|
| > loss_text_ce: 0.03892311826348305 (0.05154821300801327) |
|
| > loss_mel_ce: 3.962580442428589 (4.589151501666035) |
|
| > loss: 1.0003758668899536 (1.1601749286808751) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2995 (0.3074529195198616) |
|
| > loader_time: 0.0086 (0.009178772328974037) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:14:19 -- STEP: 22800/30063 -- GLOBAL_STEP: 22800[0m |
|
| > loss_text_ce: 0.037390001118183136 (0.05152115329577199) |
|
| > loss_mel_ce: 3.642117738723755 (4.587928225973174) |
|
| > loss: 0.9198769330978394 (1.1598623448218108) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2908 (0.3074492886505598) |
|
| > loader_time: 0.0087 (0.009178131787400403) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:14:46 -- STEP: 22850/30063 -- GLOBAL_STEP: 22850[0m |
|
| > loss_text_ce: 0.039994265884160995 (0.051493873844163554) |
|
| > loss_mel_ce: 3.982442617416382 (4.586804814578645) |
|
| > loss: 1.0056092739105225 (1.15957467211153) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2533 (0.30746673986739276) |
|
| > loader_time: 0.0088 (0.009177784564980172) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:15:11 -- STEP: 22900/30063 -- GLOBAL_STEP: 22900[0m |
|
| > loss_text_ce: 0.04023214429616928 (0.05146728353696867) |
|
| > loss_mel_ce: 4.516982078552246 (4.585872951126497) |
|
| > loss: 1.1393035650253296 (1.1593350586672466) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3547 (0.3074554911034565) |
|
| > loader_time: 0.0098 (0.009177307083096468) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:15:37 -- STEP: 22950/30063 -- GLOBAL_STEP: 22950[0m |
|
| > loss_text_ce: 0.0337962880730629 (0.05144212600923169) |
|
| > loss_mel_ce: 3.939302444458008 (4.584859523949786) |
|
| > loss: 0.9932746887207031 (1.1590754124983127) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3011 (0.30746196927587666) |
|
| > loader_time: 0.0087 (0.009177019393521868) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:16:05 -- STEP: 23000/30063 -- GLOBAL_STEP: 23000[0m |
|
| > loss_text_ce: 0.03473746404051781 (0.051417191869538637) |
|
| > loss_mel_ce: 3.8611998558044434 (4.583839598987401) |
|
| > loss: 0.9739843010902405 (1.1588141976905826) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2039 (0.30752189105489697) |
|
| > loader_time: 0.0087 (0.009177071167075005) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:16:31 -- STEP: 23050/30063 -- GLOBAL_STEP: 23050[0m |
|
| > loss_text_ce: 0.03577152639627457 (0.05139038101767951) |
|
| > loss_mel_ce: 4.094619274139404 (4.582444404500669) |
|
| > loss: 1.0325976610183716 (1.1584586963710368) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2498 (0.30753614650113703) |
|
| > loader_time: 0.0086 (0.00917686929930305) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:16:58 -- STEP: 23100/30063 -- GLOBAL_STEP: 23100[0m |
|
| > loss_text_ce: 0.034574348479509354 (0.05136337273185697) |
|
| > loss_mel_ce: 3.6973581314086914 (4.581135695950795) |
|
| > loss: 0.9329831004142761 (1.158124767162576) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2458 (0.3075535496488766) |
|
| > loader_time: 0.0084 (0.009176862642362404) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:17:25 -- STEP: 23150/30063 -- GLOBAL_STEP: 23150[0m |
|
| > loss_text_ce: 0.040144145488739014 (0.051336818907498284) |
|
| > loss_mel_ce: 5.011302471160889 (4.580022749179883) |
|
| > loss: 1.2628616094589233 (1.1578398920109598) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3814 (0.30759605905149445) |
|
| > loader_time: 0.009 (0.009176449023878992) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:17:52 -- STEP: 23200/30063 -- GLOBAL_STEP: 23200[0m |
|
| > loss_text_ce: 0.03834669664502144 (0.05131158359485248) |
|
| > loss_mel_ce: 3.7772135734558105 (4.578856437195981) |
|
| > loss: 0.9538900852203369 (1.1575420051967213) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2024 (0.30761116517000925) |
|
| > loader_time: 0.0088 (0.009175946301427262) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:18:18 -- STEP: 23250/30063 -- GLOBAL_STEP: 23250[0m |
|
| > loss_text_ce: 0.03806125745177269 (0.051285548965376425) |
|
| > loss_mel_ce: 3.950366735458374 (4.57773058886168) |
|
| > loss: 0.9971069693565369 (1.1572540344525344) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2943 (0.307623027893802) |
|
| > loader_time: 0.0093 (0.009175439260339101) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:18:43 -- STEP: 23300/30063 -- GLOBAL_STEP: 23300[0m |
|
| > loss_text_ce: 0.04195563867688179 (0.05126084982717293) |
|
| > loss_mel_ce: 4.597232818603516 (4.576497005843289) |
|
| > loss: 1.1597970724105835 (1.1569394639095965) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2758 (0.3076116613257058) |
|
| > loader_time: 0.0089 (0.009175371274415908) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:19:10 -- STEP: 23350/30063 -- GLOBAL_STEP: 23350[0m |
|
| > loss_text_ce: 0.03810012713074684 (0.051236621901249224) |
|
| > loss_mel_ce: 4.161343574523926 (4.575368474637661) |
|
| > loss: 1.049860954284668 (1.1566512741419621) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.368 (0.307621268838067) |
|
| > loader_time: 0.0092 (0.009175081396000682) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:19:35 -- STEP: 23400/30063 -- GLOBAL_STEP: 23400[0m |
|
| > loss_text_ce: 0.03793520852923393 (0.05120980165365466) |
|
| > loss_mel_ce: 4.491229057312012 (4.574332157139072) |
|
| > loss: 1.1322910785675049 (1.1563854897047656) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2723 (0.3076002011747415) |
|
| > loader_time: 0.0082 (0.009174830556934592) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:20:00 -- STEP: 23450/30063 -- GLOBAL_STEP: 23450[0m |
|
| > loss_text_ce: 0.04030856862664223 (0.05118276008243932) |
|
| > loss_mel_ce: 4.5825114250183105 (4.573169192728952) |
|
| > loss: 1.1557049751281738 (1.1560879882058042) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2767 (0.30757550726566985) |
|
| > loader_time: 0.0099 (0.00917470395183755) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:20:25 -- STEP: 23500/30063 -- GLOBAL_STEP: 23500[0m |
|
| > loss_text_ce: 0.04668625816702843 (0.0511576143480202) |
|
| > loss_mel_ce: 4.422684192657471 (4.571909707789714) |
|
| > loss: 1.1173425912857056 (1.1557668305356392) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2299 (0.30754784616510644) |
|
| > loader_time: 0.0105 (0.009174406051635624) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:20:50 -- STEP: 23550/30063 -- GLOBAL_STEP: 23550[0m |
|
| > loss_text_ce: 0.03873784467577934 (0.051131246718805305) |
|
| > loss_mel_ce: 3.663515329360962 (4.570699267964443) |
|
| > loss: 0.9255632758140564 (1.1554576286743088) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2851 (0.30754387547762) |
|
| > loader_time: 0.0086 (0.009173715322133971) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:21:16 -- STEP: 23600/30063 -- GLOBAL_STEP: 23600[0m |
|
| > loss_text_ce: 0.037237897515296936 (0.05110584732658075) |
|
| > loss_mel_ce: 3.71714448928833 (4.5692363411895185) |
|
| > loss: 0.9385955929756165 (1.1550855471294752) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.2029 (0.30752015347197637) |
|
| > loader_time: 0.0081 (0.009173140434895415) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:21:40 -- STEP: 23650/30063 -- GLOBAL_STEP: 23650[0m |
|
| > loss_text_ce: 0.04424761235713959 (0.0510818488758064) |
|
| > loss_mel_ce: 4.385425567626953 (4.568124616856799) |
|
| > loss: 1.1074182987213135 (1.1548016164368289) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.337 (0.30748736388587283) |
|
| > loader_time: 0.0087 (0.009172417457209456) |
|
|
|
|
|
[1m --> TIME: 2024-10-26 11:22:06 -- STEP: 23700/30063 -- GLOBAL_STEP: 23700[0m |
|
| > loss_text_ce: 0.043674536049366 (0.05105697165255104) |
|
| > loss_mel_ce: 4.3470916748046875 (4.567111886402716) |
|
| > loss: 1.097691535949707 (1.1545422145334416) |
|
| > current_lr: 5e-06 |
|
| > step_time: 0.3495 (0.30748174791094385) |
|
| > loader_time: 0.0098 (0.009171878899199916) |
|
|
|
! Run is kept in /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804 |
|
|