diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,4319 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 4 + | > Num. of Torch Threads: 1 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=/kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804 + + > Model has 522165080 parameters + + > EPOCH: 0/3 + --> /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804 + + > TRAINING (2024-10-26 07:55:23)  + + --> TIME: 2024-10-26 07:55:28 -- STEP: 0/30063 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.23170489072799683 (0.23170489072799683) + | > loss_mel_ce: 6.881418704986572 (6.881418704986572) + | > loss: 1.7782808542251587 (1.7782808542251587) + | > current_lr: 5e-06 + | > step_time: 0.8919 (0.8918666839599609) + | > loader_time: 3.9073 (3.9073474407196045) + + + --> TIME: 2024-10-26 07:55:55 -- STEP: 50/30063 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.21212132275104523 (0.22003370970487596) + | > loss_mel_ce: 6.219595909118652 (6.497453689575195) + | > loss: 1.6079293489456177 (1.6793718528747559) + | > current_lr: 5e-06 + | > step_time: 0.3578 (0.321729998588562) + | > loader_time: 0.0161 (0.014391927719116212) + + + --> TIME: 2024-10-26 07:56:21 -- STEP: 100/30063 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.2021128386259079 (0.21027226582169534) + | > loss_mel_ce: 6.183087348937988 (6.344042897224426) + | > loss: 1.5963000059127808 (1.6385787904262543) + | > current_lr: 5e-06 + | > step_time: 0.2329 (0.3162407755851746) + | > loader_time: 0.0091 (0.013851547241210937) + + + --> TIME: 2024-10-26 07:56:48 -- STEP: 150/30063 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.17603746056556702 (0.20275207271178564) + | > loss_mel_ce: 6.267411231994629 (6.232032321294149) + | > loss: 1.610862135887146 (1.6086960975329079) + | > current_lr: 5e-06 + | > step_time: 0.3582 (0.3162189785639445) + | > loader_time: 0.0146 (0.013357210159301757) + + + --> TIME: 2024-10-26 07:57:15 -- STEP: 200/30063 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.16569343209266663 (0.19600607939064507) + | > loss_mel_ce: 5.862248420715332 (6.15806046485901) + | > loss: 1.5069854259490967 (1.588516635298729) + | > current_lr: 5e-06 + | > step_time: 0.2549 (0.3156870305538176) + | > loader_time: 0.0104 (0.012890539169311523) + + + --> TIME: 2024-10-26 07:57:41 -- STEP: 250/30063 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.17059586942195892 (0.19009394341707234) + | > loss_mel_ce: 6.081897735595703 (6.086897583007813) + | > loss: 1.563123345375061 (1.5692478795051574) + | > current_lr: 5e-06 + | > step_time: 0.2152 (0.3150361776351927) + | > loader_time: 0.0088 (0.012582942008972168) + + + --> TIME: 2024-10-26 07:58:06 -- STEP: 300/30063 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.1637687385082245 (0.18456800366441412) + | > loss_mel_ce: 5.864283561706543 (6.0412895059585585) + | > loss: 1.5070130825042725 (1.5564643752574918) + | > current_lr: 5e-06 + | > step_time: 0.1896 (0.31275563160578396) + | > loader_time: 0.0087 (0.012389524777730304) + + + --> TIME: 2024-10-26 07:58:33 -- STEP: 350/30063 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.13504108786582947 (0.1791493505239487) + | > loss_mel_ce: 5.591071128845215 (5.996060019901823) + | > loss: 1.431528091430664 (1.5438023417336606) + | > current_lr: 5e-06 + | > step_time: 0.253 (0.3122274426051545) + | > loader_time: 0.0093 (0.012196938650948659) + + + --> TIME: 2024-10-26 07:58:58 -- STEP: 400/30063 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.15164980292320251 (0.17355694336816677) + | > loss_mel_ce: 5.829759120941162 (5.959608466625216) + | > loss: 1.4953522682189941 (1.5332913517951972) + | > current_lr: 5e-06 + | > step_time: 0.2888 (0.3111474370956418) + | > loader_time: 0.0127 (0.012029411792755124) + + + --> TIME: 2024-10-26 07:59:25 -- STEP: 450/30063 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.10973162204027176 (0.16831256224049468) + | > loss_mel_ce: 5.542464733123779 (5.920952761967979) + | > loss: 1.4130491018295288 (1.5223163302739466) + | > current_lr: 5e-06 + | > step_time: 0.3624 (0.3114083841111923) + | > loader_time: 0.0126 (0.011909065246582029) + + + --> TIME: 2024-10-26 07:59:50 -- STEP: 500/30063 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.12962713837623596 (0.16308557140827185) + | > loss_mel_ce: 5.6521897315979 (5.893536402702333) + | > loss: 1.4454542398452759 (1.5141554930210117) + | > current_lr: 5e-06 + | > step_time: 0.2397 (0.3100767297744749) + | > loader_time: 0.0091 (0.011745912075042723) + + + --> TIME: 2024-10-26 08:00:15 -- STEP: 550/30063 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.10157814621925354 (0.15845132323828617) + | > loss_mel_ce: 5.6893205642700195 (5.8659697064486425) + | > loss: 1.44772469997406 (1.5061052573810925) + | > current_lr: 5e-06 + | > step_time: 0.2883 (0.3090375276045364) + | > loader_time: 0.0082 (0.011590709252790972) + + + --> TIME: 2024-10-26 08:00:41 -- STEP: 600/30063 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.0999237596988678 (0.15431842969109616) + | > loss_mel_ce: 5.71621036529541 (5.843980432351433) + | > loss: 1.4540334939956665 (1.4995747152964278) + | > current_lr: 5e-06 + | > step_time: 0.1883 (0.308835740884145) + | > loader_time: 0.0115 (0.011461192369461066) + + + --> TIME: 2024-10-26 08:01:07 -- STEP: 650/30063 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.09502509236335754 (0.15062232117240248) + | > loss_mel_ce: 5.5468363761901855 (5.824419993620656) + | > loss: 1.4104653596878052 (1.4937605780821586) + | > current_lr: 5e-06 + | > step_time: 0.2258 (0.30844192541562604) + | > loader_time: 0.0103 (0.011378423250638531) + + + --> TIME: 2024-10-26 08:01:33 -- STEP: 700/30063 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.10775484889745712 (0.1472807513816016) + | > loss_mel_ce: 5.3456807136535645 (5.807394227981571) + | > loss: 1.3633588552474976 (1.4886687440531594) + | > current_lr: 5e-06 + | > step_time: 0.3332 (0.3085846209526063) + | > loader_time: 0.0092 (0.011265302045004711) + + + --> TIME: 2024-10-26 08:01:59 -- STEP: 750/30063 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.08867921680212021 (0.14417442302902544) + | > loss_mel_ce: 5.032188415527344 (5.788773053487144) + | > loss: 1.280216932296753 (1.4832368680636088) + | > current_lr: 5e-06 + | > step_time: 0.3739 (0.30869903278350835) + | > loader_time: 0.0157 (0.011200013478597006) + + + --> TIME: 2024-10-26 08:02:25 -- STEP: 800/30063 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.10235428810119629 (0.1412876050826164) + | > loss_mel_ce: 5.799641132354736 (5.769806320071223) + | > loss: 1.475498914718628 (1.4777734805643559) + | > current_lr: 5e-06 + | > step_time: 0.2885 (0.30898899078369135) + | > loader_time: 0.0092 (0.011131909787654876) + + + --> TIME: 2024-10-26 08:02:52 -- STEP: 850/30063 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.09865696728229523 (0.13870997198364332) + | > loss_mel_ce: 5.661565780639648 (5.752769415799311) + | > loss: 1.4400557279586792 (1.4728698461196001) + | > current_lr: 5e-06 + | > step_time: 0.3096 (0.30945486798005933) + | > loader_time: 0.0083 (0.011041099043453441) + + + --> TIME: 2024-10-26 08:03:18 -- STEP: 900/30063 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.09370041638612747 (0.13638660865525415) + | > loss_mel_ce: 5.630788803100586 (5.737670992745295) + | > loss: 1.4311223030090332 (1.4685143992635938) + | > current_lr: 5e-06 + | > step_time: 0.2504 (0.3097612884309556) + | > loader_time: 0.009 (0.010977994600931804) + + + --> TIME: 2024-10-26 08:03:44 -- STEP: 950/30063 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.08709386736154556 (0.13417530608804598) + | > loss_mel_ce: 5.793758869171143 (5.72142197608948) + | > loss: 1.4702131748199463 (1.4638993195483558) + | > current_lr: 5e-06 + | > step_time: 0.4417 (0.3091524294803016) + | > loader_time: 0.0118 (0.010924491882324219) + + + --> TIME: 2024-10-26 08:04:10 -- STEP: 1000/30063 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.08800945430994034 (0.1321571603417396) + | > loss_mel_ce: 5.326340198516846 (5.705877014636995) + | > loss: 1.3535873889923096 (1.4595085427761079) + | > current_lr: 5e-06 + | > step_time: 0.1732 (0.30914154839515673) + | > loader_time: 0.011 (0.010848899841308594) + + + --> TIME: 2024-10-26 08:04:36 -- STEP: 1050/30063 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.08601665496826172 (0.13022883801942775) + | > loss_mel_ce: 5.632408142089844 (5.693870799200877) + | > loss: 1.4296061992645264 (1.4560249086788724) + | > current_lr: 5e-06 + | > step_time: 0.2905 (0.30942515191577713) + | > loader_time: 0.0091 (0.010805808703104654) + + + --> TIME: 2024-10-26 08:05:02 -- STEP: 1100/30063 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.08602312207221985 (0.1284687467596747) + | > loss_mel_ce: 5.536700248718262 (5.681885940378363) + | > loss: 1.4056808948516846 (1.4525886709039857) + | > current_lr: 5e-06 + | > step_time: 0.1894 (0.3091714057055385) + | > loader_time: 0.0087 (0.010750555992126463) + + + --> TIME: 2024-10-26 08:05:28 -- STEP: 1150/30063 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.08417614549398422 (0.12675709724426268) + | > loss_mel_ce: 5.5014472007751465 (5.671958470966501) + | > loss: 1.396405816078186 (1.4496788912234102) + | > current_lr: 5e-06 + | > step_time: 0.2121 (0.30936030201289955) + | > loader_time: 0.0093 (0.010705902472786282) + + + --> TIME: 2024-10-26 08:05:53 -- STEP: 1200/30063 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.08322088420391083 (0.12516221993292367) + | > loss_mel_ce: 5.500393390655518 (5.660428640445076) + | > loss: 1.3959035873413086 (1.4463977145155273) + | > current_lr: 5e-06 + | > step_time: 0.2519 (0.30891982714335103) + | > loader_time: 0.0085 (0.010658098657925922) + + + --> TIME: 2024-10-26 08:06:20 -- STEP: 1250/30063 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.08325894922018051 (0.12357091580629351) + | > loss_mel_ce: 5.557662487030029 (5.648966109466552) + | > loss: 1.4102303981781006 (1.443134255504609) + | > current_lr: 5e-06 + | > step_time: 0.3241 (0.3090733589172363) + | > loader_time: 0.0088 (0.010603929901123046) + + + --> TIME: 2024-10-26 08:06:45 -- STEP: 1300/30063 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.09614972770214081 (0.12214811073472868) + | > loss_mel_ce: 4.303592681884766 (5.636168299821704) + | > loss: 1.0999356508255005 (1.4395791020760178) + | > current_lr: 5e-06 + | > step_time: 0.2189 (0.30887425972865196) + | > loader_time: 0.0091 (0.010556736542628361) + + + --> TIME: 2024-10-26 08:07:11 -- STEP: 1350/30063 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.07779347896575928 (0.12072566411009543) + | > loss_mel_ce: 5.503547668457031 (5.625556886814254) + | > loss: 1.39533531665802 (1.4365706370494997) + | > current_lr: 5e-06 + | > step_time: 0.4998 (0.30886782187002704) + | > loader_time: 0.0104 (0.010519071155124246) + + + --> TIME: 2024-10-26 08:07:36 -- STEP: 1400/30063 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.07625886052846909 (0.11942704156573332) + | > loss_mel_ce: 5.332442760467529 (5.615913562093456) + | > loss: 1.3521753549575806 (1.4338351504291813) + | > current_lr: 5e-06 + | > step_time: 0.3527 (0.3083442379747119) + | > loader_time: 0.0148 (0.010485467229570672) + + + --> TIME: 2024-10-26 08:08:02 -- STEP: 1450/30063 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.08856235444545746 (0.11822888348123124) + | > loss_mel_ce: 5.069084167480469 (5.604444723458121) + | > loss: 1.2894116640090942 (1.4306684013070738) + | > current_lr: 5e-06 + | > step_time: 0.3104 (0.30840879358094336) + | > loader_time: 0.0091 (0.010447566098180322) + + + --> TIME: 2024-10-26 08:08:28 -- STEP: 1500/30063 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.07408756762742996 (0.11703336800634863) + | > loss_mel_ce: 5.485511302947998 (5.595548205375667) + | > loss: 1.389899730682373 (1.4281453928947454) + | > current_lr: 5e-06 + | > step_time: 0.223 (0.3080576645533245) + | > loader_time: 0.0086 (0.010408626079559334) + + + --> TIME: 2024-10-26 08:08:54 -- STEP: 1550/30063 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.07401573657989502 (0.11589358337944555) + | > loss_mel_ce: 5.453428268432617 (5.586482695917924) + | > loss: 1.3818609714508057 (1.4255940691886415) + | > current_lr: 5e-06 + | > step_time: 0.3012 (0.30811007007475855) + | > loader_time: 0.0088 (0.010374061830582166) + + + --> TIME: 2024-10-26 08:09:20 -- STEP: 1600/30063 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.07491308450698853 (0.1147708569839597) + | > loss_mel_ce: 5.363227367401123 (5.577283202111716) + | > loss: 1.3595350980758667 (1.423013514205814) + | > current_lr: 5e-06 + | > step_time: 0.2884 (0.3082011656463145) + | > loader_time: 0.0125 (0.010345256626605997) + + + --> TIME: 2024-10-26 08:09:45 -- STEP: 1650/30063 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.08233626931905746 (0.11368184279311788) + | > loss_mel_ce: 5.224329471588135 (5.568112325957323) + | > loss: 1.3266664743423462 (1.4204485415689878) + | > current_lr: 5e-06 + | > step_time: 0.3383 (0.30761829665212925) + | > loader_time: 0.0099 (0.010329303596958979) + + + --> TIME: 2024-10-26 08:10:11 -- STEP: 1700/30063 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.07145506888628006 (0.11268149183953512) + | > loss_mel_ce: 5.418025970458984 (5.562068278929762) + | > loss: 1.3723702430725098 (1.418687442050261) + | > current_lr: 5e-06 + | > step_time: 0.2625 (0.30786982157651116) + | > loader_time: 0.0089 (0.010314668907838718) + + + --> TIME: 2024-10-26 08:10:36 -- STEP: 1750/30063 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.07609575986862183 (0.11171010547450612) + | > loss_mel_ce: 5.219825267791748 (5.55399457114083) + | > loss: 1.3239802122116089 (1.4164261685098927) + | > current_lr: 5e-06 + | > step_time: 0.2758 (0.30738476508004325) + | > loader_time: 0.0111 (0.010299347741263261) + + + --> TIME: 2024-10-26 08:11:02 -- STEP: 1800/30063 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.07980663329362869 (0.11075186309715114) + | > loss_mel_ce: 5.36020565032959 (5.548088555600904) + | > loss: 1.360003113746643 (1.4147101041343482) + | > current_lr: 5e-06 + | > step_time: 0.1887 (0.30729703691270616) + | > loader_time: 0.0085 (0.010279223124186205) + + + --> TIME: 2024-10-26 08:11:27 -- STEP: 1850/30063 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.0747850239276886 (0.10978481787684803) + | > loss_mel_ce: 4.964337348937988 (5.540190805744476) + | > loss: 1.2597806453704834 (1.4124939053123067) + | > current_lr: 5e-06 + | > step_time: 0.2834 (0.30689654724017995) + | > loader_time: 0.0086 (0.010249082719957513) + + + --> TIME: 2024-10-26 08:11:54 -- STEP: 1900/30063 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.07936915755271912 (0.10889650933248433) + | > loss_mel_ce: 4.668835639953613 (5.5313092786387354) + | > loss: 1.1870511770248413 (1.4100514463374494) + | > current_lr: 5e-06 + | > step_time: 0.253 (0.30733379602432254) + | > loader_time: 0.0085 (0.010230237308301431) + + + --> TIME: 2024-10-26 08:12:19 -- STEP: 1950/30063 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.0797397643327713 (0.1079766525137119) + | > loss_mel_ce: 5.299657344818115 (5.525013010073927) + | > loss: 1.3448492288589478 (1.4082474148579138) + | > current_lr: 5e-06 + | > step_time: 0.2582 (0.3070032523228572) + | > loader_time: 0.009 (0.01021249575492664) + + + --> TIME: 2024-10-26 08:12:43 -- STEP: 2000/30063 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.0669279620051384 (0.10712874733656647) + | > loss_mel_ce: 5.1178178787231445 (5.517483374118801) + | > loss: 1.2961864471435547 (1.4061530295610432) + | > current_lr: 5e-06 + | > step_time: 0.2542 (0.3064018021821976) + | > loader_time: 0.0087 (0.010187365174293524) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_2000.pth + + --> TIME: 2024-10-26 08:13:21 -- STEP: 2050/30063 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.06811168044805527 (0.10628551141881362) + | > loss_mel_ce: 4.559199810028076 (5.510345500853) + | > loss: 1.1568279266357422 (1.4041577524092144) + | > current_lr: 5e-06 + | > step_time: 0.2848 (0.30655269320418194) + | > loader_time: 0.0088 (0.010183970986343018) + + + --> TIME: 2024-10-26 08:13:47 -- STEP: 2100/30063 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.07062087208032608 (0.10543713966295834) + | > loss_mel_ce: 5.500833511352539 (5.503551035381494) + | > loss: 1.3928636312484741 (1.4022470431100758) + | > current_lr: 5e-06 + | > step_time: 0.2453 (0.30659473873320076) + | > loader_time: 0.009 (0.010166156746092303) + + + --> TIME: 2024-10-26 08:14:13 -- STEP: 2150/30063 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.06449185311794281 (0.10463515500862933) + | > loss_mel_ce: 5.3209991455078125 (5.4979623597167215) + | > loss: 1.3463727235794067 (1.4006493780224818) + | > current_lr: 5e-06 + | > step_time: 0.2544 (0.30670245226039444) + | > loader_time: 0.0085 (0.010146812061930818) + + + --> TIME: 2024-10-26 08:14:40 -- STEP: 2200/30063 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.07365085184574127 (0.10385430124842311) + | > loss_mel_ce: 5.484344005584717 (5.4911244722929835) + | > loss: 1.3894987106323242 (1.3987446926127773) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.3070101110501721) + | > loader_time: 0.0084 (0.010129004066640687) + + + --> TIME: 2024-10-26 08:15:06 -- STEP: 2250/30063 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.07574379444122314 (0.10312586959534226) + | > loss_mel_ce: 5.031594276428223 (5.484766653378802) + | > loss: 1.276834487915039 (1.3969731299877162) + | > current_lr: 5e-06 + | > step_time: 0.3132 (0.30696369330088286) + | > loader_time: 0.0089 (0.010111840777926985) + + + --> TIME: 2024-10-26 08:15:32 -- STEP: 2300/30063 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.07440293580293655 (0.10237585138529544) + | > loss_mel_ce: 4.590554714202881 (5.478043002563973) + | > loss: 1.1662393808364868 (1.395104712532914) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.30696037862611825) + | > loader_time: 0.0119 (0.010098232600999924) + + + --> TIME: 2024-10-26 08:15:57 -- STEP: 2350/30063 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.06387393921613693 (0.10166311325545013) + | > loss_mel_ce: 5.345189571380615 (5.472762038149732) + | > loss: 1.3522658348083496 (1.39360628688589) + | > current_lr: 5e-06 + | > step_time: 0.3617 (0.30689309039014423) + | > loader_time: 0.0092 (0.010082799627425828) + + + --> TIME: 2024-10-26 08:16:23 -- STEP: 2400/30063 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.06882283836603165 (0.10095946935005495) + | > loss_mel_ce: 5.448531150817871 (5.4681817411383005) + | > loss: 1.3793385028839111 (1.3922853016604977) + | > current_lr: 5e-06 + | > step_time: 0.2752 (0.30689351518948865) + | > loader_time: 0.0092 (0.01006083836158117) + + + --> TIME: 2024-10-26 08:16:50 -- STEP: 2450/30063 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.06075205281376839 (0.1002809236578796) + | > loss_mel_ce: 5.319052219390869 (5.461250479270002) + | > loss: 1.3449510335922241 (1.3903828499755075) + | > current_lr: 5e-06 + | > step_time: 0.2589 (0.30705850912600147) + | > loader_time: 0.0089 (0.010044105685487095) + + + --> TIME: 2024-10-26 08:17:14 -- STEP: 2500/30063 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.06374214589595795 (0.09958283366113915) + | > loss_mel_ce: 4.876726150512695 (5.454756479358678) + | > loss: 1.2351170778274536 (1.388584827423095) + | > current_lr: 5e-06 + | > step_time: 0.2705 (0.30664564971923836) + | > loader_time: 0.0093 (0.010029112148284926) + + + --> TIME: 2024-10-26 08:17:40 -- STEP: 2550/30063 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.05650924891233444 (0.09893472672239254) + | > loss_mel_ce: 4.584807395935059 (5.448961325907244) + | > loss: 1.1603291034698486 (1.3869740123374776) + | > current_lr: 5e-06 + | > step_time: 0.2553 (0.306775053996666) + | > loader_time: 0.0085 (0.010015119571311812) + + + --> TIME: 2024-10-26 08:18:07 -- STEP: 2600/30063 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.061880555003881454 (0.09828463365108939) + | > loss_mel_ce: 5.471291542053223 (5.443524964864442) + | > loss: 1.3832930326461792 (1.3854523987036467) + | > current_lr: 5e-06 + | > step_time: 0.3232 (0.3070918783774744) + | > loader_time: 0.0098 (0.010002823242774387) + + + --> TIME: 2024-10-26 08:18:34 -- STEP: 2650/30063 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.06976098567247391 (0.09766374249925046) + | > loss_mel_ce: 5.448820114135742 (5.437433556700658) + | > loss: 1.3796452283859253 (1.383774323913284) + | > current_lr: 5e-06 + | > step_time: 0.4661 (0.3071978550137215) + | > loader_time: 0.0091 (0.009987944566978614) + + + --> TIME: 2024-10-26 08:19:00 -- STEP: 2700/30063 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.05855318531394005 (0.09703293374153216) + | > loss_mel_ce: 4.8042755126953125 (5.431825043095487) + | > loss: 1.2157071828842163 (1.3822144935307652) + | > current_lr: 5e-06 + | > step_time: 0.2083 (0.307398458145283) + | > loader_time: 0.0094 (0.009979187117682578) + + + --> TIME: 2024-10-26 08:19:25 -- STEP: 2750/30063 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.06550798565149307 (0.09644117969816397) + | > loss_mel_ce: 4.832806587219238 (5.4272411524165785) + | > loss: 1.224578619003296 (1.3809205824245072) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.30720100134069267) + | > loader_time: 0.01 (0.00997125833684749) + + + --> TIME: 2024-10-26 08:19:49 -- STEP: 2800/30063 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.05796299874782562 (0.09583733841377726) + | > loss_mel_ce: 5.177628517150879 (5.42183058338506) + | > loss: 1.308897852897644 (1.3794169797641855) + | > current_lr: 5e-06 + | > step_time: 0.2318 (0.3068307880844391) + | > loader_time: 0.0087 (0.009960115977696018) + + + --> TIME: 2024-10-26 08:20:15 -- STEP: 2850/30063 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.07211673259735107 (0.09525114514195099) + | > loss_mel_ce: 4.86887788772583 (5.41721613423866) + | > loss: 1.2352486848831177 (1.37811681923113) + | > current_lr: 5e-06 + | > step_time: 0.2765 (0.30675500585321813) + | > loader_time: 0.0084 (0.00994930735805579) + + + --> TIME: 2024-10-26 08:20:40 -- STEP: 2900/30063 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.06425466388463974 (0.09470726624516593) + | > loss_mel_ce: 5.489013671875 (5.411897129765875) + | > loss: 1.3883171081542969 (1.3766510983582163) + | > current_lr: 5e-06 + | > step_time: 0.2555 (0.30660548086824113) + | > loader_time: 0.0101 (0.009935188951163463) + + + --> TIME: 2024-10-26 08:21:05 -- STEP: 2950/30063 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.06667879223823547 (0.0941485169353123) + | > loss_mel_ce: 4.914728164672852 (5.40638057328887) + | > loss: 1.245351791381836 (1.3751322717989867) + | > current_lr: 5e-06 + | > step_time: 0.2765 (0.3064562492855527) + | > loader_time: 0.0091 (0.009921366885557023) + + + --> TIME: 2024-10-26 08:21:32 -- STEP: 3000/30063 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.05804649740457535 (0.09361584443226471) + | > loss_mel_ce: 5.580506801605225 (5.402270357211437) + | > loss: 1.4096382856369019 (1.3739715496698972) + | > current_lr: 5e-06 + | > step_time: 0.2514 (0.30669864741961217) + | > loader_time: 0.0091 (0.009911420663197858) + + + --> TIME: 2024-10-26 08:21:58 -- STEP: 3050/30063 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.05688001587986946 (0.09307807677837683) + | > loss_mel_ce: 5.228001594543457 (5.39718115830031) + | > loss: 1.3212203979492188 (1.3725648081888877) + | > current_lr: 5e-06 + | > step_time: 0.284 (0.30661608742885976) + | > loader_time: 0.009 (0.00989944919210968) + + + --> TIME: 2024-10-26 08:22:24 -- STEP: 3100/30063 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.0635627955198288 (0.09255758854650696) + | > loss_mel_ce: 5.0295538902282715 (5.391927004245025) + | > loss: 1.2732791900634766 (1.371121147717195) + | > current_lr: 5e-06 + | > step_time: 0.2254 (0.3066209843850914) + | > loader_time: 0.0085 (0.009887698235050344) + + + --> TIME: 2024-10-26 08:22:50 -- STEP: 3150/30063 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.05579125136137009 (0.09203267107289004) + | > loss_mel_ce: 4.962165355682373 (5.38733748125652) + | > loss: 1.2544891834259033 (1.369842537622599) + | > current_lr: 5e-06 + | > step_time: 0.3112 (0.3066844329379864) + | > loader_time: 0.0091 (0.009876208229670468) + + + --> TIME: 2024-10-26 08:23:17 -- STEP: 3200/30063 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.062379587441682816 (0.09152061841683476) + | > loss_mel_ce: 5.582221031188965 (5.382664303705101) + | > loss: 1.4111500978469849 (1.36854623008519) + | > current_lr: 5e-06 + | > step_time: 0.2542 (0.30686685062944974) + | > loader_time: 0.0092 (0.009864901453256629) + + + --> TIME: 2024-10-26 08:23:42 -- STEP: 3250/30063 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.056953128427267075 (0.09102743740035951) + | > loss_mel_ce: 5.310308933258057 (5.377411424563486) + | > loss: 1.34181547164917 (1.36710971494821) + | > current_lr: 5e-06 + | > step_time: 0.2888 (0.30684112130678687) + | > loader_time: 0.0085 (0.00985392827254077) + + + --> TIME: 2024-10-26 08:24:08 -- STEP: 3300/30063 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.0671331137418747 (0.09057583946847568) + | > loss_mel_ce: 5.217059135437012 (5.372881090641026) + | > loss: 1.3210480213165283 (1.3658642319476928) + | > current_lr: 5e-06 + | > step_time: 0.3195 (0.30684599435690746) + | > loader_time: 0.0104 (0.009841780951528855) + + + --> TIME: 2024-10-26 08:24:35 -- STEP: 3350/30063 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.05577133968472481 (0.09010711561721665) + | > loss_mel_ce: 4.900613784790039 (5.368441508350092) + | > loss: 1.2390962839126587 (1.3646371554616634) + | > current_lr: 5e-06 + | > step_time: 0.3985 (0.30712972420365126) + | > loader_time: 0.0086 (0.009833562480869595) + + + --> TIME: 2024-10-26 08:25:01 -- STEP: 3400/30063 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.06137121841311455 (0.0896515914600563) + | > loss_mel_ce: 5.19257116317749 (5.364051356105247) + | > loss: 1.3134856224060059 (1.3634257363221187) + | > current_lr: 5e-06 + | > step_time: 0.2472 (0.3071863407247213) + | > loader_time: 0.0087 (0.009822374091428885) + + + --> TIME: 2024-10-26 08:25:27 -- STEP: 3450/30063 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.059456028044223785 (0.08919331718182232) + | > loss_mel_ce: 4.8889312744140625 (5.360089647251631) + | > loss: 1.2370967864990234 (1.3623207403956943) + | > current_lr: 5e-06 + | > step_time: 0.3239 (0.3072151103227041) + | > loader_time: 0.0092 (0.009815978658372094) + + + --> TIME: 2024-10-26 08:25:54 -- STEP: 3500/30063 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.059247225522994995 (0.08876542833447469) + | > loss_mel_ce: 4.851404190063477 (5.355922462122785) + | > loss: 1.2276628017425537 (1.3611719719682385) + | > current_lr: 5e-06 + | > step_time: 0.2737 (0.3073270673070641) + | > loader_time: 0.009 (0.009805948938642246) + + + --> TIME: 2024-10-26 08:26:20 -- STEP: 3550/30063 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.06208595633506775 (0.08831732398922187) + | > loss_mel_ce: 5.375740051269531 (5.3517854431985175) + | > loss: 1.3594565391540527 (1.3600256911129982) + | > current_lr: 5e-06 + | > step_time: 0.3805 (0.30733932448105133) + | > loader_time: 0.0098 (0.009795999459817392) + + + --> TIME: 2024-10-26 08:26:47 -- STEP: 3600/30063 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.06425635516643524 (0.08790372783628615) + | > loss_mel_ce: 4.582530975341797 (5.347199749350552) + | > loss: 1.1616967916488647 (1.35877586851517) + | > current_lr: 5e-06 + | > step_time: 0.3591 (0.3075332593917852) + | > loader_time: 0.0103 (0.00979210257530214) + + + --> TIME: 2024-10-26 08:27:13 -- STEP: 3650/30063 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.05682969465851784 (0.08749756801822427) + | > loss_mel_ce: 5.275410175323486 (5.3434848795198455) + | > loss: 1.3330600261688232 (1.3577456111777284) + | > current_lr: 5e-06 + | > step_time: 0.3366 (0.30768314054567575) + | > loader_time: 0.0087 (0.009783586279986671) + + + --> TIME: 2024-10-26 08:27:39 -- STEP: 3700/30063 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.05106568709015846 (0.08708993616539092) + | > loss_mel_ce: 4.455092430114746 (5.3391103113664204) + | > loss: 1.1265395879745483 (1.3565500611227876) + | > current_lr: 5e-06 + | > step_time: 0.2017 (0.30768884987444545) + | > loader_time: 0.0091 (0.009777096542152214) + + + --> TIME: 2024-10-26 08:28:05 -- STEP: 3750/30063 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.06302288919687271 (0.08669127986232451) + | > loss_mel_ce: 5.117846488952637 (5.335145044390364) + | > loss: 1.2952173948287964 (1.3554590803782112) + | > current_lr: 5e-06 + | > step_time: 0.4454 (0.30763209616343234) + | > loader_time: 0.011 (0.009769464810689306) + + + --> TIME: 2024-10-26 08:28:31 -- STEP: 3800/30063 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.05548869073390961 (0.08628803236998238) + | > loss_mel_ce: 5.010197162628174 (5.330449713343072) + | > loss: 1.2664214372634888 (1.3541844356687411) + | > current_lr: 5e-06 + | > step_time: 0.3444 (0.30778440425270537) + | > loader_time: 0.0101 (0.009762681973607931) + + + --> TIME: 2024-10-26 08:28:57 -- STEP: 3850/30063 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.05964277312159538 (0.0859230433181897) + | > loss_mel_ce: 4.641628265380859 (5.325238827854012) + | > loss: 1.1753177642822266 (1.3527904669959792) + | > current_lr: 5e-06 + | > step_time: 0.3151 (0.30771857373126194) + | > loader_time: 0.0087 (0.00975645052922238) + + + --> TIME: 2024-10-26 08:29:23 -- STEP: 3900/30063 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.06471210718154907 (0.08553808374760254) + | > loss_mel_ce: 5.375728607177734 (5.321331409001965) + | > loss: 1.3601101636886597 (1.3517173723991067) + | > current_lr: 5e-06 + | > step_time: 0.3196 (0.30783938994774507) + | > loader_time: 0.0094 (0.00975006311367721) + + + --> TIME: 2024-10-26 08:29:49 -- STEP: 3950/30063 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.060686301440000534 (0.0851611256401374) + | > loss_mel_ce: 4.59771728515625 (5.317355869691587) + | > loss: 1.1646008491516113 (1.3506292479551258) + | > current_lr: 5e-06 + | > step_time: 0.3386 (0.3079400326330456) + | > loader_time: 0.0091 (0.009744126042233252) + + + --> TIME: 2024-10-26 08:30:16 -- STEP: 4000/30063 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.05586684122681618 (0.08479565447755169) + | > loss_mel_ce: 4.8931989669799805 (5.314127012073997) + | > loss: 1.2372664213180542 (1.3497306658029524) + | > current_lr: 5e-06 + | > step_time: 0.2088 (0.30810717231035284) + | > loader_time: 0.0083 (0.009735280275344862) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_4000.pth + + --> TIME: 2024-10-26 08:30:56 -- STEP: 4050/30063 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.052190251648426056 (0.08444626804303251) + | > loss_mel_ce: 4.670748710632324 (5.309610804334103) + | > loss: 1.1807347536087036 (1.3485142673533608) + | > current_lr: 5e-06 + | > step_time: 0.2956 (0.30827877904162043) + | > loader_time: 0.0083 (0.009738614235395281) + + + --> TIME: 2024-10-26 08:31:22 -- STEP: 4100/30063 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.05852193757891655 (0.0841007058595013) + | > loss_mel_ce: 5.107022285461426 (5.305386142032906) + | > loss: 1.2913860082626343 (1.3473717111930583) + | > current_lr: 5e-06 + | > step_time: 0.2485 (0.30832245239397427) + | > loader_time: 0.0085 (0.00973010848208173) + + + --> TIME: 2024-10-26 08:31:48 -- STEP: 4150/30063 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.05039399117231369 (0.08375424219901315) + | > loss_mel_ce: 4.78805685043335 (5.301750245956057) + | > loss: 1.2096127271652222 (1.3463761213170446) + | > current_lr: 5e-06 + | > step_time: 0.3087 (0.3082484912872319) + | > loader_time: 0.0088 (0.009720529016241984) + + + --> TIME: 2024-10-26 08:32:14 -- STEP: 4200/30063 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.05939150229096413 (0.0834163748171359) + | > loss_mel_ce: 4.867889881134033 (5.298356269200647) + | > loss: 1.2318203449249268 (1.345443160292645) + | > current_lr: 5e-06 + | > step_time: 0.2443 (0.3082044482231145) + | > loader_time: 0.0086 (0.00971100239526659) + + + --> TIME: 2024-10-26 08:32:40 -- STEP: 4250/30063 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.053591351956129074 (0.0830794850412538) + | > loss_mel_ce: 4.552379131317139 (5.294268745310171) + | > loss: 1.1514925956726074 (1.3443370568471764) + | > current_lr: 5e-06 + | > step_time: 0.2516 (0.30825748174330814) + | > loader_time: 0.0091 (0.009706943231470458) + + + --> TIME: 2024-10-26 08:33:06 -- STEP: 4300/30063 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.061251912266016006 (0.08275208839756816) + | > loss_mel_ce: 4.968867301940918 (5.290114767606872) + | > loss: 1.257529854774475 (1.3432167131817583) + | > current_lr: 5e-06 + | > step_time: 0.2238 (0.3082856611872833) + | > loader_time: 0.0085 (0.009699547623479101) + + + --> TIME: 2024-10-26 08:33:33 -- STEP: 4350/30063 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.05414481461048126 (0.08243993345262682) + | > loss_mel_ce: 5.35730504989624 (5.28703852357536) + | > loss: 1.3528624773025513 (1.3423696133871155) + | > current_lr: 5e-06 + | > step_time: 0.3017 (0.3084913237889613) + | > loader_time: 0.0091 (0.009692860142938036) + + + --> TIME: 2024-10-26 08:33:59 -- STEP: 4400/30063 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.05556824058294296 (0.08212703285230837) + | > loss_mel_ce: 5.031979084014893 (5.283835186633201) + | > loss: 1.2718868255615234 (1.341490553956136) + | > current_lr: 5e-06 + | > step_time: 0.2063 (0.3085539044033404) + | > loader_time: 0.0088 (0.009686627875674867) + + + --> TIME: 2024-10-26 08:34:25 -- STEP: 4450/30063 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.052425116300582886 (0.08182114905520776) + | > loss_mel_ce: 5.110321998596191 (5.279547021469374) + | > loss: 1.2906867265701294 (1.3403420417362344) + | > current_lr: 5e-06 + | > step_time: 0.4415 (0.30855855175618424) + | > loader_time: 0.009 (0.009681190640738854) + + + --> TIME: 2024-10-26 08:34:51 -- STEP: 4500/30063 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.05320815369486809 (0.08151754578948027) + | > loss_mel_ce: 5.224514961242676 (5.276120165877871) + | > loss: 1.3194308280944824 (1.3394094270202805) + | > current_lr: 5e-06 + | > step_time: 0.2894 (0.30852230554156895) + | > loader_time: 0.0092 (0.009674134254455577) + + + --> TIME: 2024-10-26 08:35:16 -- STEP: 4550/30063 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.05023062974214554 (0.0812040988519625) + | > loss_mel_ce: 5.157011032104492 (5.272658821514675) + | > loss: 1.301810383796692 (1.3384657292182585) + | > current_lr: 5e-06 + | > step_time: 0.2756 (0.30835745062146935) + | > loader_time: 0.0091 (0.00966801947289772) + + + --> TIME: 2024-10-26 08:35:41 -- STEP: 4600/30063 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.053258538246154785 (0.08090741043262516) + | > loss_mel_ce: 4.84642219543457 (5.26945495237475) + | > loss: 1.2249201536178589 (1.3375905898472498) + | > current_lr: 5e-06 + | > step_time: 0.2202 (0.30824133598286263) + | > loader_time: 0.0091 (0.009662288116372158) + + + --> TIME: 2024-10-26 08:36:07 -- STEP: 4650/30063 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.05655081942677498 (0.0806219706476056) + | > loss_mel_ce: 5.497552394866943 (5.265216525344438) + | > loss: 1.3885258436203003 (1.3364596231650252) + | > current_lr: 5e-06 + | > step_time: 0.3771 (0.3082388992207032) + | > loader_time: 0.0089 (0.009654397605567862) + + + --> TIME: 2024-10-26 08:36:32 -- STEP: 4700/30063 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.05432265251874924 (0.08034848345879557) + | > loss_mel_ce: 4.778601169586182 (5.261200698639489) + | > loss: 1.208230972290039 (1.3353872947363112) + | > current_lr: 5e-06 + | > step_time: 0.2468 (0.30811896993758897) + | > loader_time: 0.0094 (0.009647638036849666) + + + --> TIME: 2024-10-26 08:36:57 -- STEP: 4750/30063 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.054096613079309464 (0.0800752571813371) + | > loss_mel_ce: 5.384282112121582 (5.258289113245517) + | > loss: 1.359594702720642 (1.3345910918712582) + | > current_lr: 5e-06 + | > step_time: 0.2872 (0.3080228351291863) + | > loader_time: 0.0083 (0.009640800827427931) + + + --> TIME: 2024-10-26 08:37:22 -- STEP: 4800/30063 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.061632998287677765 (0.079796956415133) + | > loss_mel_ce: 5.053612232208252 (5.25527614369989) + | > loss: 1.2788113355636597 (1.333768274200456) + | > current_lr: 5e-06 + | > step_time: 0.1771 (0.30790709629654955) + | > loader_time: 0.0114 (0.009634631226460159) + + + --> TIME: 2024-10-26 08:37:47 -- STEP: 4850/30063 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.05280528962612152 (0.07951196558082228) + | > loss_mel_ce: 4.944235801696777 (5.251706095282568) + | > loss: 1.2492603063583374 (1.3328045143909035) + | > current_lr: 5e-06 + | > step_time: 0.5044 (0.30781903458624843) + | > loader_time: 0.0123 (0.009628454680295358) + + + --> TIME: 2024-10-26 08:38:13 -- STEP: 4900/30063 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.05001556873321533 (0.07923891535142852) + | > loss_mel_ce: 5.33284330368042 (5.248878259804786) + | > loss: 1.3457146883010864 (1.3320292929848816) + | > current_lr: 5e-06 + | > step_time: 0.3974 (0.307817061288017) + | > loader_time: 0.0103 (0.00962495181025293) + + + --> TIME: 2024-10-26 08:38:38 -- STEP: 4950/30063 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.057402871549129486 (0.07898112643622987) + | > loss_mel_ce: 4.846681594848633 (5.246179242615752) + | > loss: 1.22602117061615 (1.33129009149291) + | > current_lr: 5e-06 + | > step_time: 0.4024 (0.3077281044950395) + | > loader_time: 0.0088 (0.009619294754182472) + + + --> TIME: 2024-10-26 08:39:04 -- STEP: 5000/30063 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.04803409054875374 (0.0787204534493386) + | > loss_mel_ce: 5.035665988922119 (5.242769097089774) + | > loss: 1.2709250450134277 (1.3303723868727653) + | > current_lr: 5e-06 + | > step_time: 0.2026 (0.30763668522834847) + | > loader_time: 0.0087 (0.00961424646377566) + + + --> TIME: 2024-10-26 08:39:30 -- STEP: 5050/30063 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.049694254994392395 (0.07846560239496794) + | > loss_mel_ce: 4.9324750900268555 (5.239026062040048) + | > loss: 1.245542287826538 (1.3293729153836094) + | > current_lr: 5e-06 + | > step_time: 0.3366 (0.307683503462537) + | > loader_time: 0.0086 (0.009608216332917195) + + + --> TIME: 2024-10-26 08:39:56 -- STEP: 5100/30063 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.050695694983005524 (0.07821530453057261) + | > loss_mel_ce: 4.998305797576904 (5.235262287691527) + | > loss: 1.2622504234313965 (1.3283693974275197) + | > current_lr: 5e-06 + | > step_time: 0.3483 (0.307639473372815) + | > loader_time: 0.0094 (0.009605188556745962) + + + --> TIME: 2024-10-26 08:40:22 -- STEP: 5150/30063 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.049359772354364395 (0.07797325653430905) + | > loss_mel_ce: 4.994658946990967 (5.232206886856298) + | > loss: 1.2610046863555908 (1.3275450352904847) + | > current_lr: 5e-06 + | > step_time: 0.3781 (0.3076929405360551) + | > loader_time: 0.0098 (0.009603027788180764) + + + --> TIME: 2024-10-26 08:40:48 -- STEP: 5200/30063 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.05708217993378639 (0.0777273942974324) + | > loss_mel_ce: 4.99580192565918 (5.228980866441358) + | > loss: 1.263221025466919 (1.326677064609066) + | > current_lr: 5e-06 + | > step_time: 0.3029 (0.30775526069677833) + | > loader_time: 0.0088 (0.009598385599943315) + + + --> TIME: 2024-10-26 08:41:14 -- STEP: 5250/30063 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.05572541058063507 (0.07748530706834218) + | > loss_mel_ce: 4.9542059898376465 (5.225410131590704) + | > loss: 1.2524828910827637 (1.3257238590944354) + | > current_lr: 5e-06 + | > step_time: 0.3069 (0.30780510725293886) + | > loader_time: 0.0087 (0.009593558084397096) + + + --> TIME: 2024-10-26 08:41:40 -- STEP: 5300/30063 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.05386144667863846 (0.07725407427422827) + | > loss_mel_ce: 4.7178850173950195 (5.22191834346303) + | > loss: 1.192936658859253 (1.3247931038995935) + | > current_lr: 5e-06 + | > step_time: 0.1883 (0.3077082051421115) + | > loader_time: 0.0083 (0.009588770416547675) + + + --> TIME: 2024-10-26 08:42:06 -- STEP: 5350/30063 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.050288598984479904 (0.07702725212150624) + | > loss_mel_ce: 5.080891132354736 (5.21858347264405) + | > loss: 1.2827949523925781 (1.323902680751318) + | > current_lr: 5e-06 + | > step_time: 0.3596 (0.30782888916051304) + | > loader_time: 0.0089 (0.009583431136942355) + + + --> TIME: 2024-10-26 08:42:32 -- STEP: 5400/30063 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.05106084793806076 (0.07679917003476489) + | > loss_mel_ce: 4.948300838470459 (5.215010965400264) + | > loss: 1.2498403787612915 (1.322952533485711) + | > current_lr: 5e-06 + | > step_time: 0.2516 (0.3078046407964498) + | > loader_time: 0.0102 (0.009578969522758768) + + + --> TIME: 2024-10-26 08:42:59 -- STEP: 5450/30063 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.05196305736899376 (0.07658719975740515) + | > loss_mel_ce: 4.577043533325195 (5.212189451576368) + | > loss: 1.1572515964508057 (1.322194162454078) + | > current_lr: 5e-06 + | > step_time: 0.2998 (0.3079055452784272) + | > loader_time: 0.0087 (0.00957394188697185) + + + --> TIME: 2024-10-26 08:43:24 -- STEP: 5500/30063 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.04803841933608055 (0.07636471346223872) + | > loss_mel_ce: 5.292352676391602 (5.208950527321206) + | > loss: 1.3350977897644043 (1.3213288097923428) + | > current_lr: 5e-06 + | > step_time: 0.2291 (0.30783557198264433) + | > loader_time: 0.0103 (0.009568363666534418) + + + --> TIME: 2024-10-26 08:43:51 -- STEP: 5550/30063 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.05387811362743378 (0.07613851040131848) + | > loss_mel_ce: 4.544703483581543 (5.206089579049529) + | > loss: 1.149645447731018 (1.320557021998067) + | > current_lr: 5e-06 + | > step_time: 0.2587 (0.3079100893209651) + | > loader_time: 0.0085 (0.009562606210107199) + + + --> TIME: 2024-10-26 08:44:16 -- STEP: 5600/30063 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.050927676260471344 (0.07591794898666947) + | > loss_mel_ce: 4.936334609985352 (5.203281921616617) + | > loss: 1.24681556224823 (1.3197999671420846) + | > current_lr: 5e-06 + | > step_time: 0.2297 (0.30788117843014906) + | > loader_time: 0.0089 (0.009557633655411853) + + + --> TIME: 2024-10-26 08:44:42 -- STEP: 5650/30063 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.049536194652318954 (0.07570459768110122) + | > loss_mel_ce: 4.739019870758057 (5.200053286214838) + | > loss: 1.197139024734497 (1.3189394704957942) + | > current_lr: 5e-06 + | > step_time: 0.2896 (0.3078766363278962) + | > loader_time: 0.0085 (0.009550985699206321) + + + --> TIME: 2024-10-26 08:45:07 -- STEP: 5700/30063 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.04668520390987396 (0.07548934124279424) + | > loss_mel_ce: 5.0926408767700195 (5.196563154061627) + | > loss: 1.2848315238952637 (1.3180131233470447) + | > current_lr: 5e-06 + | > step_time: 0.2395 (0.30773977877801006) + | > loader_time: 0.0091 (0.009546013380351808) + + + --> TIME: 2024-10-26 08:45:32 -- STEP: 5750/30063 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.0504402332007885 (0.07528607162703622) + | > loss_mel_ce: 4.741614818572998 (5.193402985489879) + | > loss: 1.1980137825012207 (1.317172263736306) + | > current_lr: 5e-06 + | > step_time: 0.3093 (0.30766080051919775) + | > loader_time: 0.0083 (0.009539858486341384) + + + --> TIME: 2024-10-26 08:45:57 -- STEP: 5800/30063 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.05087360739707947 (0.07507364639883908) + | > loss_mel_ce: 4.553732872009277 (5.190338049387101) + | > loss: 1.1511516571044922 (1.3163529234199642) + | > current_lr: 5e-06 + | > step_time: 0.2069 (0.3076071765916111) + | > loader_time: 0.0087 (0.00953650865061529) + + + --> TIME: 2024-10-26 08:46:22 -- STEP: 5850/30063 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.04882040247321129 (0.07487271299219522) + | > loss_mel_ce: 5.060157775878906 (5.187742671436724) + | > loss: 1.2772445678710938 (1.3156538455812288) + | > current_lr: 5e-06 + | > step_time: 0.2981 (0.30755144706139315) + | > loader_time: 0.0113 (0.009532623331770932) + + + --> TIME: 2024-10-26 08:46:49 -- STEP: 5900/30063 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.048869259655475616 (0.0746715444730498) + | > loss_mel_ce: 4.846346378326416 (5.184533368894604) + | > loss: 1.2238038778305054 (1.3148012278544638) + | > current_lr: 5e-06 + | > step_time: 0.3048 (0.30758367595026187) + | > loader_time: 0.0089 (0.009527626967026005) + + + --> TIME: 2024-10-26 08:47:14 -- STEP: 5950/30063 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.047826722264289856 (0.07446985409602887) + | > loss_mel_ce: 4.583195686340332 (5.180775097157772) + | > loss: 1.1577556133270264 (1.313811237341209) + | > current_lr: 5e-06 + | > step_time: 0.231 (0.3075581868356028) + | > loader_time: 0.0086 (0.009524337183527583) + + + --> TIME: 2024-10-26 08:47:39 -- STEP: 6000/30063 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.04876580461859703 (0.0742642337394256) + | > loss_mel_ce: 4.826101303100586 (5.177349878668782) + | > loss: 1.2187167406082153 (1.3129035276075134) + | > current_lr: 5e-06 + | > step_time: 0.1881 (0.3074538544813807) + | > loader_time: 0.0084 (0.00952050487200419) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_6000.pth + + --> TIME: 2024-10-26 08:48:19 -- STEP: 6050/30063 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.054143842309713364 (0.07407664288235588) + | > loss_mel_ce: 4.892397403717041 (5.174122432953061) + | > loss: 1.2366353273391724 (1.3120497684340762) + | > current_lr: 5e-06 + | > step_time: 0.2324 (0.30756384439705) + | > loader_time: 0.0083 (0.00951935362224737) + + + --> TIME: 2024-10-26 08:48:46 -- STEP: 6100/30063 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.045703545212745667 (0.07389048568904391) + | > loss_mel_ce: 4.344536781311035 (5.170498295729271) + | > loss: 1.0975600481033325 (1.3110971948260097) + | > current_lr: 5e-06 + | > step_time: 0.23 (0.30757187655714857) + | > loader_time: 0.0087 (0.00951429894713105) + + + --> TIME: 2024-10-26 08:49:12 -- STEP: 6150/30063 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.04807350039482117 (0.07370351681076896) + | > loss_mel_ce: 4.908357620239258 (5.167895246017263) + | > loss: 1.2391077280044556 (1.310399690188037) + | > current_lr: 5e-06 + | > step_time: 0.401 (0.3076262278285471) + | > loader_time: 0.0087 (0.009509249392563736) + + + --> TIME: 2024-10-26 08:49:36 -- STEP: 6200/30063 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.04511073976755142 (0.0735146028890965) + | > loss_mel_ce: 4.413634300231934 (5.1647894039077045) + | > loss: 1.1146862506866455 (1.30957600121536) + | > current_lr: 5e-06 + | > step_time: 0.2195 (0.30744003030561784) + | > loader_time: 0.0082 (0.009503971107544439) + + + --> TIME: 2024-10-26 08:50:02 -- STEP: 6250/30063 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.05372001975774765 (0.07332887484490866) + | > loss_mel_ce: 5.218628883361816 (5.161942618141165) + | > loss: 1.3180872201919556 (1.3088178728008226) + | > current_lr: 5e-06 + | > step_time: 0.3007 (0.3074469510269182) + | > loader_time: 0.0082 (0.009500380287170413) + + + --> TIME: 2024-10-26 08:50:26 -- STEP: 6300/30063 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.050848305225372314 (0.07314446925229964) + | > loss_mel_ce: 4.92686128616333 (5.158804636039424) + | > loss: 1.2444274425506592 (1.3079872758899374) + | > current_lr: 5e-06 + | > step_time: 0.2759 (0.30729720603852195) + | > loader_time: 0.0085 (0.009494357298290913) + + + --> TIME: 2024-10-26 08:50:52 -- STEP: 6350/30063 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.05048312246799469 (0.07296217626767357) + | > loss_mel_ce: 4.990358352661133 (5.1554244580982225) + | > loss: 1.260210394859314 (1.3070966581284487) + | > current_lr: 5e-06 + | > step_time: 0.337 (0.30731659449930415) + | > loader_time: 0.0086 (0.00949081413389191) + + + --> TIME: 2024-10-26 08:51:20 -- STEP: 6400/30063 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.05153748020529747 (0.0727860816090832) + | > loss_mel_ce: 5.4646196365356445 (5.1535943691804995) + | > loss: 1.3790392875671387 (1.3065951122529762) + | > current_lr: 5e-06 + | > step_time: 0.3188 (0.3074952596798555) + | > loader_time: 0.0093 (0.009487704224884512) + + + --> TIME: 2024-10-26 08:51:46 -- STEP: 6450/30063 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.04921098053455353 (0.0726089353946058) + | > loss_mel_ce: 4.615845203399658 (5.150817711297854) + | > loss: 1.1662640571594238 (1.3058566612605857) + | > current_lr: 5e-06 + | > step_time: 0.2673 (0.3075618502520791) + | > loader_time: 0.0091 (0.009482960516168162) + + + --> TIME: 2024-10-26 08:52:12 -- STEP: 6500/30063 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.05639159679412842 (0.07243927258539651) + | > loss_mel_ce: 4.94861364364624 (5.148278458118438) + | > loss: 1.2512513399124146 (1.3051794322270571) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.30757776249372215) + | > loader_time: 0.0098 (0.00947986932901235) + + + --> TIME: 2024-10-26 08:52:38 -- STEP: 6550/30063 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.051986660808324814 (0.07226459981262218) + | > loss_mel_ce: 4.255697250366211 (5.1454107075247135) + | > loss: 1.076920986175537 (1.3044188263944045) + | > current_lr: 5e-06 + | > step_time: 0.2862 (0.30757568388494977) + | > loader_time: 0.0086 (0.009478177041497844) + + + --> TIME: 2024-10-26 08:53:04 -- STEP: 6600/30063 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.04714253917336464 (0.07208799334242931) + | > loss_mel_ce: 4.200692653656006 (5.142138416478127) + | > loss: 1.0619587898254395 (1.303556602001186) + | > current_lr: 5e-06 + | > step_time: 0.2395 (0.30760203650503604) + | > loader_time: 0.0092 (0.00947571837540828) + + + --> TIME: 2024-10-26 08:53:32 -- STEP: 6650/30063 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.05410097911953926 (0.07192627017995468) + | > loss_mel_ce: 5.115808486938477 (5.139731081482162) + | > loss: 1.2924773693084717 (1.302914337556161) + | > current_lr: 5e-06 + | > step_time: 0.2998 (0.3077805868665093) + | > loader_time: 0.0085 (0.009473313245558196) + + + --> TIME: 2024-10-26 08:53:59 -- STEP: 6700/30063 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.04336101934313774 (0.07175930735391017) + | > loss_mel_ce: 5.180060863494873 (5.1371050313693365) + | > loss: 1.3058555126190186 (1.3022160842881234) + | > current_lr: 5e-06 + | > step_time: 0.4085 (0.3078166410460415) + | > loader_time: 0.0099 (0.009471960743861406) + + + --> TIME: 2024-10-26 08:54:24 -- STEP: 6750/30063 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.047823090106248856 (0.07159254969104564) + | > loss_mel_ce: 4.645129680633545 (5.1344579917766415) + | > loss: 1.1732381582260132 (1.3015126349749349) + | > current_lr: 5e-06 + | > step_time: 0.2598 (0.30775234702781534) + | > loader_time: 0.0091 (0.009469611591762957) + + + --> TIME: 2024-10-26 08:54:49 -- STEP: 6800/30063 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.04295102506875992 (0.0714244175959816) + | > loss_mel_ce: 5.38251256942749 (5.131899861307704) + | > loss: 1.3563659191131592 (1.3008310693765348) + | > current_lr: 5e-06 + | > step_time: 0.2092 (0.30767295865451727) + | > loader_time: 0.0097 (0.009467599181567914) + + + --> TIME: 2024-10-26 08:55:14 -- STEP: 6850/30063 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.040934547781944275 (0.07126103512072648) + | > loss_mel_ce: 4.77911901473999 (5.129008587816335) + | > loss: 1.205013394355774 (1.3000674053613246) + | > current_lr: 5e-06 + | > step_time: 0.2337 (0.30758801564683147) + | > loader_time: 0.0085 (0.009463490430456) + + + --> TIME: 2024-10-26 08:55:40 -- STEP: 6900/30063 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.05115421488881111 (0.07110199623511751) + | > loss_mel_ce: 4.8123369216918945 (5.12636384487152) + | > loss: 1.2158727645874023 (1.2993664598724077) + | > current_lr: 5e-06 + | > step_time: 0.2231 (0.30756727101146447) + | > loader_time: 0.0089 (0.00945973040401071) + + + --> TIME: 2024-10-26 08:56:05 -- STEP: 6950/30063 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.05364755541086197 (0.07094418296282241) + | > loss_mel_ce: 4.720203876495361 (5.123611506180796) + | > loss: 1.19346284866333 (1.2986389218988996) + | > current_lr: 5e-06 + | > step_time: 0.29 (0.3075364296727915) + | > loader_time: 0.009 (0.009459517053562952) + + + --> TIME: 2024-10-26 08:56:31 -- STEP: 7000/30063 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.05403774976730347 (0.07079904444249618) + | > loss_mel_ce: 4.834413051605225 (5.120805518422807) + | > loss: 1.2221126556396484 (1.297901140366278) + | > current_lr: 5e-06 + | > step_time: 0.3045 (0.3074961162294674) + | > loader_time: 0.0088 (0.009458149433135978) + + + --> TIME: 2024-10-26 08:56:57 -- STEP: 7050/30063 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.05605943873524666 (0.07064579639451728) + | > loss_mel_ce: 4.523179531097412 (5.11837789062067) + | > loss: 1.1448097229003906 (1.2972559213807358) + | > current_lr: 5e-06 + | > step_time: 0.4236 (0.30749494637158037) + | > loader_time: 0.009 (0.009454369308255234) + + + --> TIME: 2024-10-26 08:57:25 -- STEP: 7100/30063 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.04986662417650223 (0.07050368365687387) + | > loss_mel_ce: 4.875361442565918 (5.11584436873315) + | > loss: 1.231307029724121 (1.2965870127207757) + | > current_lr: 5e-06 + | > step_time: 0.2078 (0.307694830659411) + | > loader_time: 0.0084 (0.0094521239441885) + + + --> TIME: 2024-10-26 08:57:50 -- STEP: 7150/30063 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.05578547716140747 (0.07034925732683468) + | > loss_mel_ce: 4.955790996551514 (5.113094672256415) + | > loss: 1.2528941631317139 (1.295860982041255) + | > current_lr: 5e-06 + | > step_time: 0.3415 (0.3076202752039996) + | > loader_time: 0.0091 (0.009448732162688987) + + + --> TIME: 2024-10-26 08:58:15 -- STEP: 7200/30063 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.05076083540916443 (0.07019972331252784) + | > loss_mel_ce: 4.482229232788086 (5.110276120073265) + | > loss: 1.1332474946975708 (1.2951189605229392) + | > current_lr: 5e-06 + | > step_time: 0.2769 (0.30749342474672664) + | > loader_time: 0.009 (0.009445868697431345) + + + --> TIME: 2024-10-26 08:58:39 -- STEP: 7250/30063 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.0449727438390255 (0.07005203231449791) + | > loss_mel_ce: 4.635336875915527 (5.107826707346685) + | > loss: 1.1700774431228638 (1.2944696846501542) + | > current_lr: 5e-06 + | > step_time: 0.2087 (0.3073897930342589) + | > loader_time: 0.0096 (0.009442852086034306) + + + --> TIME: 2024-10-26 08:59:05 -- STEP: 7300/30063 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.048395540565252304 (0.06990216300554883) + | > loss_mel_ce: 5.156826019287109 (5.104894975113541) + | > loss: 1.3013054132461548 (1.2936992842530515) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.307364421544011) + | > loader_time: 0.0088 (0.00944006825146609) + + + --> TIME: 2024-10-26 08:59:31 -- STEP: 7350/30063 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.044843968003988266 (0.06975436385036732) + | > loss_mel_ce: 4.734724521636963 (5.102093723777199) + | > loss: 1.194892168045044 (1.2929620216006288) + | > current_lr: 5e-06 + | > step_time: 0.3598 (0.3073976058051713) + | > loader_time: 0.0086 (0.009438275934076626) + + + --> TIME: 2024-10-26 08:59:57 -- STEP: 7400/30063 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.04968007281422615 (0.0696153411556136) + | > loss_mel_ce: 5.202208042144775 (5.099391696968593) + | > loss: 1.312972068786621 (1.292251759227865) + | > current_lr: 5e-06 + | > step_time: 0.2706 (0.30741635164699166) + | > loader_time: 0.009 (0.00943746038385339) + + + --> TIME: 2024-10-26 09:00:23 -- STEP: 7450/30063 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.048379987478256226 (0.06948040632993582) + | > loss_mel_ce: 4.924644470214844 (5.096992545351885) + | > loss: 1.2432560920715332 (1.2916182375514231) + | > current_lr: 5e-06 + | > step_time: 0.3054 (0.3074529529417934) + | > loader_time: 0.0098 (0.009436228099285349) + + + --> TIME: 2024-10-26 09:00:50 -- STEP: 7500/30063 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.04645639285445213 (0.06934010148098071) + | > loss_mel_ce: 4.962783336639404 (5.094743250052133) + | > loss: 1.2523099184036255 (1.291020837537444) + | > current_lr: 5e-06 + | > step_time: 0.3205 (0.3074977883338941) + | > loader_time: 0.0086 (0.009433635552724196) + + + --> TIME: 2024-10-26 09:01:15 -- STEP: 7550/30063 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.04705824330449104 (0.06920398574347134) + | > loss_mel_ce: 5.064231872558594 (5.092304064422253) + | > loss: 1.277822494506836 (1.2903770122701723) + | > current_lr: 5e-06 + | > step_time: 0.3818 (0.3074381890202213) + | > loader_time: 0.01 (0.009430843283798514) + + + --> TIME: 2024-10-26 09:01:40 -- STEP: 7600/30063 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.04761073365807533 (0.06906843905288138) + | > loss_mel_ce: 5.081933975219727 (5.089699823542644) + | > loss: 1.2823861837387085 (1.2896920653942348) + | > current_lr: 5e-06 + | > step_time: 0.2875 (0.3073494929702671) + | > loader_time: 0.0089 (0.009430322490240391) + + + --> TIME: 2024-10-26 09:02:07 -- STEP: 7650/30063 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.04644805192947388 (0.06892944757065742) + | > loss_mel_ce: 5.051183700561523 (5.08739608789581) + | > loss: 1.274407982826233 (1.2890813836474788) + | > current_lr: 5e-06 + | > step_time: 0.3589 (0.3074791598475849) + | > loader_time: 0.0088 (0.009428170802546475) + + + --> TIME: 2024-10-26 09:02:33 -- STEP: 7700/30063 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.0478401780128479 (0.06879798726892317) + | > loss_mel_ce: 4.146048069000244 (5.084770686812214) + | > loss: 1.0484720468521118 (1.288392168283459) + | > current_lr: 5e-06 + | > step_time: 0.2478 (0.30750714106993365) + | > loader_time: 0.0089 (0.00942642502970509) + + + --> TIME: 2024-10-26 09:02:59 -- STEP: 7750/30063 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.04729926958680153 (0.06866483971092009) + | > loss_mel_ce: 4.452090263366699 (5.081776839994614) + | > loss: 1.124847412109375 (1.2876104197348281) + | > current_lr: 5e-06 + | > step_time: 0.2975 (0.30749182821089344) + | > loader_time: 0.0099 (0.009424184737666954) + + + --> TIME: 2024-10-26 09:03:25 -- STEP: 7800/30063 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.04890133813023567 (0.06852941507187027) + | > loss_mel_ce: 4.190849304199219 (5.079268223658585) + | > loss: 1.0599377155303955 (1.286949409452765) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.3075223493270397) + | > loader_time: 0.0086 (0.009422722504689136) + + + --> TIME: 2024-10-26 09:03:52 -- STEP: 7850/30063 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.04709719866514206 (0.06839888308220989) + | > loss_mel_ce: 4.656945705413818 (5.076435230370539) + | > loss: 1.1760107278823853 (1.2862085281663604) + | > current_lr: 5e-06 + | > step_time: 0.1628 (0.3075831049718687) + | > loader_time: 0.0148 (0.009422303157247548) + + + --> TIME: 2024-10-26 09:04:17 -- STEP: 7900/30063 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.05796726420521736 (0.06827097406775891) + | > loss_mel_ce: 4.585546016693115 (5.0731271419042265) + | > loss: 1.160878300666809 (1.2853495288121515) + | > current_lr: 5e-06 + | > step_time: 0.1843 (0.3075533524344251) + | > loader_time: 0.0084 (0.009420771719534178) + + + --> TIME: 2024-10-26 09:04:43 -- STEP: 7950/30063 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.04201190546154976 (0.06814106286977822) + | > loss_mel_ce: 4.645221710205078 (5.070994603798823) + | > loss: 1.1718083620071411 (1.2847839164958776) + | > current_lr: 5e-06 + | > step_time: 0.3145 (0.3075668084546467) + | > loader_time: 0.0095 (0.00941868362186839) + + + --> TIME: 2024-10-26 09:05:11 -- STEP: 8000/30063 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.05196619778871536 (0.06801372760813683) + | > loss_mel_ce: 4.982473373413086 (5.068517632007598) + | > loss: 1.2586098909378052 (1.2841328397467697) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.3077086949050438) + | > loader_time: 0.0096 (0.009416900187730783) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_8000.pth + + --> TIME: 2024-10-26 09:05:50 -- STEP: 8050/30063 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.04565136879682541 (0.06789201162385274) + | > loss_mel_ce: 4.208895206451416 (5.066229526211756) + | > loss: 1.0636366605758667 (1.2835303842873274) + | > current_lr: 5e-06 + | > step_time: 0.3615 (0.3077111552516878) + | > loader_time: 0.0091 (0.009418802972165687) + + + --> TIME: 2024-10-26 09:06:17 -- STEP: 8100/30063 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.04153524711728096 (0.06776188364183461) + | > loss_mel_ce: 4.566102027893066 (5.064151602262331) + | > loss: 1.151909351348877 (1.2829783713008116) + | > current_lr: 5e-06 + | > step_time: 0.2512 (0.3077945953239641) + | > loader_time: 0.0091 (0.009417468206382085) + + + --> TIME: 2024-10-26 09:06:42 -- STEP: 8150/30063 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.044901292771101 (0.06763758468573079) + | > loss_mel_ce: 5.0430474281311035 (5.061811980908633) + | > loss: 1.2719871997833252 (1.282362391180784) + | > current_lr: 5e-06 + | > step_time: 0.3351 (0.3077500936297563) + | > loader_time: 0.0084 (0.009414776936630523) + + + --> TIME: 2024-10-26 09:07:09 -- STEP: 8200/30063 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.0454082153737545 (0.06751905771790118) + | > loss_mel_ce: 4.519289493560791 (5.059019703138165) + | > loss: 1.1411744356155396 (1.2816346900056015) + | > current_lr: 5e-06 + | > step_time: 0.2869 (0.3078093310972551) + | > loader_time: 0.0092 (0.009412290468448539) + + + --> TIME: 2024-10-26 09:07:34 -- STEP: 8250/30063 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.04671340435743332 (0.06739923784407702) + | > loss_mel_ce: 4.8445916175842285 (5.056781493013554) + | > loss: 1.2228262424468994 (1.2810451825026277) + | > current_lr: 5e-06 + | > step_time: 0.2844 (0.30781348384510504) + | > loader_time: 0.0082 (0.009410045768275399) + + + --> TIME: 2024-10-26 09:08:00 -- STEP: 8300/30063 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.05163051187992096 (0.06727993517665258) + | > loss_mel_ce: 4.690332412719727 (5.054585831624914) + | > loss: 1.1854907274246216 (1.2804664414546547) + | > current_lr: 5e-06 + | > step_time: 0.2079 (0.3078222240310129) + | > loader_time: 0.0085 (0.009408139177115563) + + + --> TIME: 2024-10-26 09:08:26 -- STEP: 8350/30063 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.04417100176215172 (0.06715430419869765) + | > loss_mel_ce: 4.941776275634766 (5.051836240334425) + | > loss: 1.246486783027649 (1.279747635904184) + | > current_lr: 5e-06 + | > step_time: 0.2839 (0.30775566469409615) + | > loader_time: 0.0095 (0.009407010306855158) + + + --> TIME: 2024-10-26 09:08:50 -- STEP: 8400/30063 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.04773438721895218 (0.06704179907306317) + | > loss_mel_ce: 5.108583450317383 (5.049579876718067) + | > loss: 1.2890794277191162 (1.2791554187663936) + | > current_lr: 5e-06 + | > step_time: 0.2894 (0.307648855050406) + | > loader_time: 0.0089 (0.009405551779837828) + + + --> TIME: 2024-10-26 09:09:15 -- STEP: 8450/30063 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.04428543895483017 (0.06692515764319332) + | > loss_mel_ce: 4.722553730010986 (5.047003762256464) + | > loss: 1.1917097568511963 (1.2784822298013219) + | > current_lr: 5e-06 + | > step_time: 0.2553 (0.3075593126455013) + | > loader_time: 0.0084 (0.009403263266975352) + + + --> TIME: 2024-10-26 09:09:40 -- STEP: 8500/30063 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.04003987833857536 (0.06680960359906447) + | > loss_mel_ce: 4.371427059173584 (5.045015277357662) + | > loss: 1.102866768836975 (1.2779562201149297) + | > current_lr: 5e-06 + | > step_time: 0.2101 (0.30753717430900224) + | > loader_time: 0.0097 (0.009401686976937676) + + + --> TIME: 2024-10-26 09:10:06 -- STEP: 8550/30063 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.05339425802230835 (0.0667010460144769) + | > loss_mel_ce: 4.546338081359863 (5.042802318104522) + | > loss: 1.149933099746704 (1.2773758409385765) + | > current_lr: 5e-06 + | > step_time: 0.4699 (0.3075226592459883) + | > loader_time: 0.0096 (0.00939881589677598) + + + --> TIME: 2024-10-26 09:10:33 -- STEP: 8600/30063 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.04726944491267204 (0.06658767819794457) + | > loss_mel_ce: 4.55790376663208 (5.040967856950533) + | > loss: 1.1512932777404785 (1.276888883675251) + | > current_lr: 5e-06 + | > step_time: 0.2092 (0.3075814642185399) + | > loader_time: 0.0087 (0.009396943075712328) + + + --> TIME: 2024-10-26 09:10:59 -- STEP: 8650/30063 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.04324669763445854 (0.06647360552209995) + | > loss_mel_ce: 4.4818572998046875 (5.038614866885141) + | > loss: 1.13127601146698 (1.276272117981329) + | > current_lr: 5e-06 + | > step_time: 0.3103 (0.30763903915537677) + | > loader_time: 0.0087 (0.00939516323839308) + + + --> TIME: 2024-10-26 09:11:25 -- STEP: 8700/30063 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.051467884331941605 (0.0663593505421418) + | > loss_mel_ce: 4.216485977172852 (5.0364380571211855) + | > loss: 1.066988468170166 (1.2756993517930455) + | > current_lr: 5e-06 + | > step_time: 0.2445 (0.30762604461319243) + | > loader_time: 0.0085 (0.009393067195497703) + + + --> TIME: 2024-10-26 09:11:50 -- STEP: 8750/30063 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.0551118329167366 (0.066253345688326) + | > loss_mel_ce: 4.701315402984619 (5.034400744301929) + | > loss: 1.189106822013855 (1.275163522311616) + | > current_lr: 5e-06 + | > step_time: 0.3043 (0.30755533133915597) + | > loader_time: 0.0091 (0.009391172845023013) + + + --> TIME: 2024-10-26 09:12:16 -- STEP: 8800/30063 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.04350483790040016 (0.06614231976976784) + | > loss_mel_ce: 4.2583112716674805 (5.031916452917182) + | > loss: 1.0754539966583252 (1.2745146929540383) + | > current_lr: 5e-06 + | > step_time: 0.2035 (0.30755023468624576) + | > loader_time: 0.0086 (0.00938868793574246) + + + --> TIME: 2024-10-26 09:12:42 -- STEP: 8850/30063 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.05048595368862152 (0.06603203036859207) + | > loss_mel_ce: 4.905280113220215 (5.029709189517347) + | > loss: 1.2389415502548218 (1.2739353047925832) + | > current_lr: 5e-06 + | > step_time: 0.2311 (0.3075820515519492) + | > loader_time: 0.0082 (0.009386972923063282) + + + --> TIME: 2024-10-26 09:13:10 -- STEP: 8900/30063 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.04182222858071327 (0.06592351649375104) + | > loss_mel_ce: 4.195362567901611 (5.027495626653174) + | > loss: 1.0592962503433228 (1.2733547856298681) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.3077082822296067) + | > loader_time: 0.0084 (0.009386230934871707) + + + --> TIME: 2024-10-26 09:13:35 -- STEP: 8950/30063 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.04534178972244263 (0.06581567967445819) + | > loss_mel_ce: 3.9669606685638428 (5.025229556840218) + | > loss: 1.0030755996704102 (1.2727613089737235) + | > current_lr: 5e-06 + | > step_time: 0.2552 (0.30766561116586283) + | > loader_time: 0.009 (0.009384270273773351) + + + --> TIME: 2024-10-26 09:14:01 -- STEP: 9000/30063 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.04479904845356941 (0.06570983089423836) + | > loss_mel_ce: 4.215223789215088 (5.0228632046381545) + | > loss: 1.0650056600570679 (1.2721432587173203) + | > current_lr: 5e-06 + | > step_time: 0.1871 (0.30765716698434703) + | > loader_time: 0.0097 (0.009382897933324166) + + + --> TIME: 2024-10-26 09:14:25 -- STEP: 9050/30063 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.039955902844667435 (0.06559978377588538) + | > loss_mel_ce: 4.754630088806152 (5.020990268354249) + | > loss: 1.1986465454101562 (1.2716475128666422) + | > current_lr: 5e-06 + | > step_time: 0.2596 (0.30754815778679717) + | > loader_time: 0.0096 (0.009380926005748082) + + + --> TIME: 2024-10-26 09:14:50 -- STEP: 9100/30063 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.04646781459450722 (0.06549145685652116) + | > loss_mel_ce: 4.84979772567749 (5.019107038555556) + | > loss: 1.2240663766860962 (1.2711496236625563) + | > current_lr: 5e-06 + | > step_time: 0.2246 (0.3074864152761617) + | > loader_time: 0.0094 (0.00937947870610833) + + + --> TIME: 2024-10-26 09:15:16 -- STEP: 9150/30063 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.048718325793743134 (0.06539056486648605) + | > loss_mel_ce: 4.25131368637085 (5.017017595937336) + | > loss: 1.0750080347061157 (1.2706020400628317) + | > current_lr: 5e-06 + | > step_time: 0.2336 (0.30749383577232264) + | > loader_time: 0.0085 (0.009377983854116628) + + + --> TIME: 2024-10-26 09:15:43 -- STEP: 9200/30063 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.0471118688583374 (0.0652840959475091) + | > loss_mel_ce: 4.605229377746582 (5.014790821593734) + | > loss: 1.1630853414535522 (1.270018729275978) + | > current_lr: 5e-06 + | > step_time: 0.2076 (0.307537568341132) + | > loader_time: 0.0091 (0.009376272999722012) + + + --> TIME: 2024-10-26 09:16:09 -- STEP: 9250/30063 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.04292181506752968 (0.06518247394827563) + | > loss_mel_ce: 4.478394508361816 (5.012488087705659) + | > loss: 1.1303291320800781 (1.26941764032518) + | > current_lr: 5e-06 + | > step_time: 0.262 (0.3075567583908919) + | > loader_time: 0.0085 (0.009373795251588551) + + + --> TIME: 2024-10-26 09:16:35 -- STEP: 9300/30063 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.04348050802946091 (0.0650859934131624) + | > loss_mel_ce: 4.4285078048706055 (5.01027032567608) + | > loss: 1.1179970502853394 (1.2688390796671578) + | > current_lr: 5e-06 + | > step_time: 0.3043 (0.3075873072685749) + | > loader_time: 0.0095 (0.00937042564474125) + + + --> TIME: 2024-10-26 09:16:59 -- STEP: 9350/30063 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.04959333688020706 (0.06498527519165821) + | > loss_mel_ce: 4.9205451011657715 (5.008190027048234) + | > loss: 1.2425346374511719 (1.2682938254580727) + | > current_lr: 5e-06 + | > step_time: 0.4285 (0.3075054761050235) + | > loader_time: 0.0087 (0.009367147154986533) + + + --> TIME: 2024-10-26 09:17:25 -- STEP: 9400/30063 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.04399513453245163 (0.06488285649726352) + | > loss_mel_ce: 4.437255382537842 (5.006124274172675) + | > loss: 1.1203125715255737 (1.267751782549183) + | > current_lr: 5e-06 + | > step_time: 0.2483 (0.3074770924639211) + | > loader_time: 0.0085 (0.009363563010033129) + + + --> TIME: 2024-10-26 09:17:51 -- STEP: 9450/30063 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.045519113540649414 (0.06478508448119817) + | > loss_mel_ce: 4.412610054016113 (5.004127570182552) + | > loss: 1.114532232284546 (1.2672281635879792) + | > current_lr: 5e-06 + | > step_time: 0.335 (0.3075090731767136) + | > loader_time: 0.0099 (0.009359977863453038) + + + --> TIME: 2024-10-26 09:18:17 -- STEP: 9500/30063 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.05057214945554733 (0.06468588528429196) + | > loss_mel_ce: 4.834811210632324 (5.0018192673482345) + | > loss: 1.2213457822799683 (1.2666262880751917) + | > current_lr: 5e-06 + | > step_time: 0.2501 (0.3075194097318165) + | > loader_time: 0.0077 (0.009355699012154014) + + + --> TIME: 2024-10-26 09:18:44 -- STEP: 9550/30063 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.04791411757469177 (0.06458384960694766) + | > loss_mel_ce: 4.45664644241333 (4.999583621299702) + | > loss: 1.1261401176452637 (1.2660418676331369) + | > current_lr: 5e-06 + | > step_time: 0.3974 (0.30755159370562346) + | > loader_time: 0.0082 (0.009351899137047563) + + + --> TIME: 2024-10-26 09:19:10 -- STEP: 9600/30063 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.046952612698078156 (0.06448913765411504) + | > loss_mel_ce: 4.986007213592529 (4.997377066314218) + | > loss: 1.258239984512329 (1.265466550911462) + | > current_lr: 5e-06 + | > step_time: 0.2717 (0.3076145727932469) + | > loader_time: 0.0083 (0.009349546507000917) + + + --> TIME: 2024-10-26 09:19:37 -- STEP: 9650/30063 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.04784470424056053 (0.06439239426559422) + | > loss_mel_ce: 4.7191996574401855 (4.994989513288506) + | > loss: 1.1917611360549927 (1.2648454768237627) + | > current_lr: 5e-06 + | > step_time: 0.4927 (0.3076365560571163) + | > loader_time: 0.0089 (0.009346292945387446) + + + --> TIME: 2024-10-26 09:20:02 -- STEP: 9700/30063 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.0444919653236866 (0.06429525005525534) + | > loss_mel_ce: 4.810189723968506 (4.992949280837148) + | > loss: 1.2136703729629517 (1.2643111326460947) + | > current_lr: 5e-06 + | > step_time: 0.2055 (0.3075870000210023) + | > loader_time: 0.0082 (0.009342599455843262) + + + --> TIME: 2024-10-26 09:20:27 -- STEP: 9750/30063 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.05829209089279175 (0.06420120623860595) + | > loss_mel_ce: 5.063032627105713 (4.990763189633688) + | > loss: 1.2803311347961426 (1.2637410988991047) + | > current_lr: 5e-06 + | > step_time: 0.3879 (0.30758349741422386) + | > loader_time: 0.0093 (0.009340144573113848) + + + --> TIME: 2024-10-26 09:20:53 -- STEP: 9800/30063 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.04380590841174126 (0.06410940858592473) + | > loss_mel_ce: 4.697413444519043 (4.98880662020372) + | > loss: 1.185304880142212 (1.2632290071674726) + | > current_lr: 5e-06 + | > step_time: 0.2079 (0.30755612587442366) + | > loader_time: 0.0085 (0.009337990892176715) + + + --> TIME: 2024-10-26 09:21:19 -- STEP: 9850/30063 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.05170084908604622 (0.06402088557856929) + | > loss_mel_ce: 4.406652450561523 (4.986744991220194) + | > loss: 1.1145883798599243 (1.2626914691501443) + | > current_lr: 5e-06 + | > step_time: 0.2504 (0.30754950961485883) + | > loader_time: 0.0085 (0.009333648173337039) + + + --> TIME: 2024-10-26 09:21:45 -- STEP: 9900/30063 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.04140520840883255 (0.06392819652954722) + | > loss_mel_ce: 4.494514465332031 (4.984458376759229) + | > loss: 1.1339799165725708 (1.2620966432672527) + | > current_lr: 5e-06 + | > step_time: 0.2764 (0.3075814427751499) + | > loader_time: 0.0086 (0.009331044741351184) + + + --> TIME: 2024-10-26 09:22:11 -- STEP: 9950/30063 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.04541175439953804 (0.06383677862359938) + | > loss_mel_ce: 4.4480977058410645 (4.982165972671318) + | > loss: 1.1233773231506348 (1.2615006877429484) + | > current_lr: 5e-06 + | > step_time: 0.261 (0.3075829853604203) + | > loader_time: 0.0085 (0.009327885829024567) + + + --> TIME: 2024-10-26 09:22:37 -- STEP: 10000/30063 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.04455357417464256 (0.06374829961173221) + | > loss_mel_ce: 4.299884796142578 (4.980085641765593) + | > loss: 1.0861096382141113 (1.2609584852993456) + | > current_lr: 5e-06 + | > step_time: 0.2243 (0.3076040564298643) + | > loader_time: 0.0088 (0.009324054002761826) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_10000.pth + + --> TIME: 2024-10-26 09:23:14 -- STEP: 10050/30063 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.04175692796707153 (0.0636585009242615) + | > loss_mel_ce: 4.36986780166626 (4.978244080211395) + | > loss: 1.1029062271118164 (1.260475645272883) + | > current_lr: 5e-06 + | > step_time: 0.3089 (0.3075430592257004) + | > loader_time: 0.0087 (0.009324142256779445) + + + --> TIME: 2024-10-26 09:23:39 -- STEP: 10100/30063 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.045051198452711105 (0.06356994441719623) + | > loss_mel_ce: 4.352637767791748 (4.97634028354494) + | > loss: 1.0994222164154053 (1.2599775569745775) + | > current_lr: 5e-06 + | > step_time: 0.2283 (0.3075051268728665) + | > loader_time: 0.0086 (0.009319894124965849) + + + --> TIME: 2024-10-26 09:24:05 -- STEP: 10150/30063 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.04328029602766037 (0.06348312948321166) + | > loss_mel_ce: 4.290757656097412 (4.974175616297235) + | > loss: 1.0835094451904297 (1.2594146864754785) + | > current_lr: 5e-06 + | > step_time: 0.2499 (0.3075034415428287) + | > loader_time: 0.0086 (0.009316884966319411) + + + --> TIME: 2024-10-26 09:24:32 -- STEP: 10200/30063 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.04577929526567459 (0.06339595576857804) + | > loss_mel_ce: 4.149557113647461 (4.972113497116996) + | > loss: 1.0488340854644775 (1.2588773632517019) + | > current_lr: 5e-06 + | > step_time: 0.2402 (0.3075460172634511) + | > loader_time: 0.0095 (0.009313935836156197) + + + --> TIME: 2024-10-26 09:24:57 -- STEP: 10250/30063 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.056496694684028625 (0.06330684762720643) + | > loss_mel_ce: 4.128530979156494 (4.9700258190806395) + | > loss: 1.0462568998336792 (1.2583331667446473) + | > current_lr: 5e-06 + | > step_time: 0.2594 (0.3075111797146692) + | > loader_time: 0.0081 (0.009311463797964683) + + + --> TIME: 2024-10-26 09:25:22 -- STEP: 10300/30063 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.04111476242542267 (0.06322176657219235) + | > loss_mel_ce: 4.747211933135986 (4.967864628879775) + | > loss: 1.1970816850662231 (1.2577715989339664) + | > current_lr: 5e-06 + | > step_time: 0.2454 (0.3075020216506673) + | > loader_time: 0.0083 (0.009308677552973167) + + + --> TIME: 2024-10-26 09:25:48 -- STEP: 10350/30063 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.05186992138624191 (0.06313835474424018) + | > loss_mel_ce: 4.420424938201904 (4.96585930842709) + | > loss: 1.1180737018585205 (1.2572494158422274) + | > current_lr: 5e-06 + | > step_time: 0.2853 (0.3075112058344682) + | > loader_time: 0.0087 (0.00930810400829222) + + + --> TIME: 2024-10-26 09:26:14 -- STEP: 10400/30063 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.05159060284495354 (0.06305225857569319) + | > loss_mel_ce: 4.625476837158203 (4.9640355559266585) + | > loss: 1.1692668199539185 (1.2567719536790443) + | > current_lr: 5e-06 + | > step_time: 0.2464 (0.30747214418191293) + | > loader_time: 0.0082 (0.009306314220795262) + + + --> TIME: 2024-10-26 09:26:39 -- STEP: 10450/30063 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.04371977597475052 (0.0629651706248663) + | > loss_mel_ce: 4.416383266448975 (4.962114855615721) + | > loss: 1.1150257587432861 (1.2562700066098715) + | > current_lr: 5e-06 + | > step_time: 0.2537 (0.3073897621392074) + | > loader_time: 0.0087 (0.009304624470797454) + + + --> TIME: 2024-10-26 09:27:04 -- STEP: 10500/30063 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.04045282304286957 (0.06288022180256378) + | > loss_mel_ce: 4.312146186828613 (4.959961118561886) + | > loss: 1.0881497859954834 (1.2557103351354564) + | > current_lr: 5e-06 + | > step_time: 0.2194 (0.3073659356889283) + | > loader_time: 0.0086 (0.009302499612172442) + + + --> TIME: 2024-10-26 09:27:30 -- STEP: 10550/30063 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.04451645910739899 (0.06279552242555311) + | > loss_mel_ce: 4.953880310058594 (4.957788552672947) + | > loss: 1.2495992183685303 (1.2551460188259975) + | > current_lr: 5e-06 + | > step_time: 0.3019 (0.3073851745162542) + | > loader_time: 0.008 (0.009301255370768334) + + + --> TIME: 2024-10-26 09:27:55 -- STEP: 10600/30063 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.04350705444812775 (0.06271199698964089) + | > loss_mel_ce: 4.969170093536377 (4.955824810491425) + | > loss: 1.253169298171997 (1.254634201914631) + | > current_lr: 5e-06 + | > step_time: 0.1848 (0.3073364001400076) + | > loader_time: 0.0083 (0.00929868232529119) + + + --> TIME: 2024-10-26 09:28:22 -- STEP: 10650/30063 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.049490541219711304 (0.06263028691527135) + | > loss_mel_ce: 4.5164361000061035 (4.953777034786395) + | > loss: 1.141481637954712 (1.2541018304690474) + | > current_lr: 5e-06 + | > step_time: 0.3596 (0.3073863224692197) + | > loader_time: 0.0089 (0.009296632775678325) + + + --> TIME: 2024-10-26 09:28:49 -- STEP: 10700/30063 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.04379856586456299 (0.06254980638901748) + | > loss_mel_ce: 4.02572774887085 (4.952053799116735) + | > loss: 1.0173815488815308 (1.2536509013844392) + | > current_lr: 5e-06 + | > step_time: 0.2009 (0.30744275610023647) + | > loader_time: 0.0087 (0.009295135948145497) + + + --> TIME: 2024-10-26 09:29:14 -- STEP: 10750/30063 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.041184332221746445 (0.06246910147923354) + | > loss_mel_ce: 4.321810245513916 (4.949850816460552) + | > loss: 1.09074866771698 (1.2530799794973295) + | > current_lr: 5e-06 + | > step_time: 0.298 (0.3074132442474375) + | > loader_time: 0.0081 (0.00929274448128634) + + + --> TIME: 2024-10-26 09:29:39 -- STEP: 10800/30063 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.05466574430465698 (0.062386307825368706) + | > loss_mel_ce: 4.5749311447143555 (4.947921558177041) + | > loss: 1.1573991775512695 (1.252576966489905) + | > current_lr: 5e-06 + | > step_time: 0.2888 (0.3073418782817005) + | > loader_time: 0.0085 (0.009290271578011702) + + + --> TIME: 2024-10-26 09:30:04 -- STEP: 10850/30063 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.04323854669928551 (0.06230753721660717) + | > loss_mel_ce: 4.261952877044678 (4.946132907340067) + | > loss: 1.0762978792190552 (1.2521101111310948) + | > current_lr: 5e-06 + | > step_time: 0.2506 (0.30733895710536524) + | > loader_time: 0.0094 (0.00928900901073686) + + + --> TIME: 2024-10-26 09:30:30 -- STEP: 10900/30063 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.04208390787243843 (0.06223016297325066) + | > loss_mel_ce: 4.531040668487549 (4.944159147411323) + | > loss: 1.143281102180481 (1.2515973275626453) + | > current_lr: 5e-06 + | > step_time: 0.2056 (0.30735979819516646) + | > loader_time: 0.009 (0.009287161848960683) + + + --> TIME: 2024-10-26 09:30:56 -- STEP: 10950/30063 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.047369055449962616 (0.062152143009857344) + | > loss_mel_ce: 4.727395057678223 (4.942128990556565) + | > loss: 1.1936910152435303 (1.2510702833574099) + | > current_lr: 5e-06 + | > step_time: 0.3379 (0.3073398395869298) + | > loader_time: 0.0086 (0.009285833171513543) + + + --> TIME: 2024-10-26 09:31:22 -- STEP: 11000/30063 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.043431513011455536 (0.06207625814120862) + | > loss_mel_ce: 4.0957818031311035 (4.940389428203769) + | > loss: 1.0348032712936401 (1.25061642155322) + | > current_lr: 5e-06 + | > step_time: 0.3057 (0.30735996122793807) + | > loader_time: 0.0089 (0.009285420417785654) + + + --> TIME: 2024-10-26 09:31:50 -- STEP: 11050/30063 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.04232507199048996 (0.06199860490487696) + | > loss_mel_ce: 4.593900680541992 (4.938755974100737) + | > loss: 1.1590564250946045 (1.250188644725271) + | > current_lr: 5e-06 + | > step_time: 0.4958 (0.3074452250467716) + | > loader_time: 0.0095 (0.009284549730395848) + + + --> TIME: 2024-10-26 09:32:16 -- STEP: 11100/30063 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.03986891359090805 (0.06192052349884484) + | > loss_mel_ce: 3.9813547134399414 (4.9369965221001335) + | > loss: 1.0053058862686157 (1.2497292614036835) + | > current_lr: 5e-06 + | > step_time: 0.2055 (0.3074579733986048) + | > loader_time: 0.0092 (0.009283515053826416) + + + --> TIME: 2024-10-26 09:32:42 -- STEP: 11150/30063 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.04339003562927246 (0.061842740150377204) + | > loss_mel_ce: 4.884430885314941 (4.935050227032663) + | > loss: 1.2319552898406982 (1.2492232418327565) + | > current_lr: 5e-06 + | > step_time: 0.3042 (0.3074926709273479) + | > loader_time: 0.0081 (0.00928223900730835) + + + --> TIME: 2024-10-26 09:33:08 -- STEP: 11200/30063 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.03811433166265488 (0.06176529086760363) + | > loss_mel_ce: 4.602193355560303 (4.933052611351026) + | > loss: 1.1600769758224487 (1.2487044755422632) + | > current_lr: 5e-06 + | > step_time: 0.225 (0.3075171173257497) + | > loader_time: 0.0092 (0.009280633181333538) + + + --> TIME: 2024-10-26 09:33:35 -- STEP: 11250/30063 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.04009737819433212 (0.06168698878122685) + | > loss_mel_ce: 4.639787673950195 (4.931229447703905) + | > loss: 1.169971227645874 (1.2482291091018243) + | > current_lr: 5e-06 + | > step_time: 0.3121 (0.30753642082214466) + | > loader_time: 0.0087 (0.009278857188754595) + + + --> TIME: 2024-10-26 09:34:00 -- STEP: 11300/30063 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.04528151825070381 (0.06161454172425814) + | > loss_mel_ce: 3.8009588718414307 (4.929205906116871) + | > loss: 0.961560070514679 (1.2477051119097553) + | > current_lr: 5e-06 + | > step_time: 0.1876 (0.3074960428634584) + | > loader_time: 0.0083 (0.009277473487685197) + + + --> TIME: 2024-10-26 09:34:26 -- STEP: 11350/30063 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.04788221791386604 (0.06154117748042808) + | > loss_mel_ce: 4.789422988891602 (4.927349509310635) + | > loss: 1.2093262672424316 (1.2472226716505794) + | > current_lr: 5e-06 + | > step_time: 0.3024 (0.30753151282339986) + | > loader_time: 0.0093 (0.009276164155699599) + + + --> TIME: 2024-10-26 09:34:52 -- STEP: 11400/30063 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.043647926300764084 (0.06146557947457345) + | > loss_mel_ce: 4.334468841552734 (4.925313280423499) + | > loss: 1.094529151916504 (1.2466947149655263) + | > current_lr: 5e-06 + | > step_time: 0.2396 (0.3074915554021542) + | > loader_time: 0.0096 (0.009274755151648257) + + + --> TIME: 2024-10-26 09:35:18 -- STEP: 11450/30063 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.04008049517869949 (0.061390320308278595) + | > loss_mel_ce: 4.669530391693115 (4.9232789974546005) + | > loss: 1.1774027347564697 (1.2461673294352642) + | > current_lr: 5e-06 + | > step_time: 0.3743 (0.30749865681844096) + | > loader_time: 0.0104 (0.009274467380806855) + + + --> TIME: 2024-10-26 09:35:44 -- STEP: 11500/30063 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.04463668540120125 (0.061317288802369634) + | > loss_mel_ce: 4.804383277893066 (4.921257623050545) + | > loss: 1.2122550010681152 (1.2456437279452441) + | > current_lr: 5e-06 + | > step_time: 0.2746 (0.3074950084686288) + | > loader_time: 0.0086 (0.009272557196409797) + + + --> TIME: 2024-10-26 09:36:08 -- STEP: 11550/30063 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.04534080997109413 (0.061246064917801116) + | > loss_mel_ce: 4.371310710906982 (4.919232475499592) + | > loss: 1.1041629314422607 (1.2451196350989393) + | > current_lr: 5e-06 + | > step_time: 0.2882 (0.3074358258722154) + | > loader_time: 0.0088 (0.009270697011576063) + + + --> TIME: 2024-10-26 09:36:34 -- STEP: 11600/30063 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.040644340217113495 (0.06117405682127779) + | > loss_mel_ce: 4.364027976989746 (4.917290277255015) + | > loss: 1.1011680364608765 (1.2446160835079068) + | > current_lr: 5e-06 + | > step_time: 0.2374 (0.30742867319748385) + | > loader_time: 0.0087 (0.00926907781896919) + + + --> TIME: 2024-10-26 09:37:00 -- STEP: 11650/30063 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.04068606719374657 (0.06110194205086616) + | > loss_mel_ce: 4.3269362449646 (4.915155055840151) + | > loss: 1.0919055938720703 (1.2440642494807403) + | > current_lr: 5e-06 + | > step_time: 0.1915 (0.30742117179821565) + | > loader_time: 0.0078 (0.009267638926853939) + + + --> TIME: 2024-10-26 09:37:26 -- STEP: 11700/30063 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.04523228853940964 (0.0610304809068766) + | > loss_mel_ce: 4.482456207275391 (4.913419318708618) + | > loss: 1.1319221258163452 (1.2436124498976584) + | > current_lr: 5e-06 + | > step_time: 0.2734 (0.30745505648800525) + | > loader_time: 0.0091 (0.00926680880734044) + + + --> TIME: 2024-10-26 09:37:52 -- STEP: 11750/30063 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.04164936766028404 (0.060957539411935255) + | > loss_mel_ce: 4.371998310089111 (4.9112012915712935) + | > loss: 1.1034119129180908 (1.2430397077266193) + | > current_lr: 5e-06 + | > step_time: 0.3082 (0.3074599346404391) + | > loader_time: 0.0084 (0.00926557502340764) + + + --> TIME: 2024-10-26 09:38:18 -- STEP: 11800/30063 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.046756986528635025 (0.060887990358080256) + | > loss_mel_ce: 4.79047155380249 (4.9093842670069066) + | > loss: 1.2093071937561035 (1.2425680643168528) + | > current_lr: 5e-06 + | > step_time: 0.1735 (0.3074478529469451) + | > loader_time: 0.0085 (0.009264048014656988) + + + --> TIME: 2024-10-26 09:38:45 -- STEP: 11850/30063 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.04690614715218544 (0.06081919361644642) + | > loss_mel_ce: 3.9194750785827637 (4.907402724475317) + | > loss: 0.9915953278541565 (1.2420554794691765) + | > current_lr: 5e-06 + | > step_time: 0.3445 (0.30751572146194484) + | > loader_time: 0.0091 (0.00926319428134067) + + + --> TIME: 2024-10-26 09:39:12 -- STEP: 11900/30063 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.043324876576662064 (0.06075021928943241) + | > loss_mel_ce: 4.400256633758545 (4.905776227041471) + | > loss: 1.1108953952789307 (1.2416316115505526) + | > current_lr: 5e-06 + | > step_time: 0.2313 (0.30754688747790737) + | > loader_time: 0.0081 (0.00926224516219453) + + + --> TIME: 2024-10-26 09:39:37 -- STEP: 11950/30063 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.04530574753880501 (0.06068029705263567) + | > loss_mel_ce: 4.623571872711182 (4.90410553742654) + | > loss: 1.1672194004058838 (1.241196458604043) + | > current_lr: 5e-06 + | > step_time: 0.2508 (0.3075305535504014) + | > loader_time: 0.009 (0.009261040607755663) + + + --> TIME: 2024-10-26 09:40:04 -- STEP: 12000/30063 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.04893041402101517 (0.06061391102833038) + | > loss_mel_ce: 4.919193267822266 (4.902676748355251) + | > loss: 1.2420309782028198 (1.2408226648469738) + | > current_lr: 5e-06 + | > step_time: 0.2711 (0.30758818437655805) + | > loader_time: 0.0088 (0.009259514431158713) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_12000.pth + + --> TIME: 2024-10-26 09:40:44 -- STEP: 12050/30063 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.043795425444841385 (0.060545098793333356) + | > loss_mel_ce: 4.284984588623047 (4.900787907082038) + | > loss: 1.0821950435638428 (1.2403332514931025) + | > current_lr: 5e-06 + | > step_time: 0.3613 (0.30757709479430934) + | > loader_time: 0.0096 (0.00925977006492775) + + + --> TIME: 2024-10-26 09:41:09 -- STEP: 12100/30063 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.04468939080834389 (0.06047493495693616) + | > loss_mel_ce: 4.306922912597656 (4.8989265521893035) + | > loss: 1.0879030227661133 (1.2398503718248053) + | > current_lr: 5e-06 + | > step_time: 0.2095 (0.30754347677073235) + | > loader_time: 0.0085 (0.00925852235683727) + + + --> TIME: 2024-10-26 09:41:35 -- STEP: 12150/30063 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.048856157809495926 (0.060406149037396814) + | > loss_mel_ce: 4.719479084014893 (4.896998180931017) + | > loss: 1.1920838356018066 (1.2393510825202263) + | > current_lr: 5e-06 + | > step_time: 0.2949 (0.3075311496718921) + | > loader_time: 0.0083 (0.009256773250093213) + + + --> TIME: 2024-10-26 09:42:01 -- STEP: 12200/30063 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.046400874853134155 (0.060341220195786735) + | > loss_mel_ce: 4.550384044647217 (4.895332270215788) + | > loss: 1.1491962671279907 (1.2389183726564752) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.30752775452176084) + | > loader_time: 0.009 (0.00925582199800213) + + + --> TIME: 2024-10-26 09:42:26 -- STEP: 12250/30063 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.04334597662091255 (0.06027286043732736) + | > loss_mel_ce: 4.602388381958008 (4.893202876635976) + | > loss: 1.1614335775375366 (1.2383689343053434) + | > current_lr: 5e-06 + | > step_time: 0.377 (0.3074975147247315) + | > loader_time: 0.0089 (0.009253975459507562) + + + --> TIME: 2024-10-26 09:42:52 -- STEP: 12300/30063 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.04300916567444801 (0.0602083785968218) + | > loss_mel_ce: 4.4165449142456055 (4.8914678343718805) + | > loss: 1.1148885488510132 (1.2379190532899456) + | > current_lr: 5e-06 + | > step_time: 0.225 (0.3074895622865943) + | > loader_time: 0.0092 (0.009253245330438408) + + + --> TIME: 2024-10-26 09:43:18 -- STEP: 12350/30063 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.038267143070697784 (0.060140637354510376) + | > loss_mel_ce: 4.305757522583008 (4.889687799712446) + | > loss: 1.0860061645507812 (1.2374571093180848) + | > current_lr: 5e-06 + | > step_time: 0.2836 (0.30750177989604527) + | > loader_time: 0.0099 (0.009252584144654092) + + + --> TIME: 2024-10-26 09:43:44 -- STEP: 12400/30063 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.04278199002146721 (0.06007430747241489) + | > loss_mel_ce: 4.11808967590332 (4.887955180387369) + | > loss: 1.0402178764343262 (1.2370073720043675) + | > current_lr: 5e-06 + | > step_time: 0.2287 (0.30753053536338215) + | > loader_time: 0.0085 (0.009251727942497526) + + + --> TIME: 2024-10-26 09:44:10 -- STEP: 12450/30063 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.04406498745083809 (0.06000631342929995) + | > loss_mel_ce: 4.708452224731445 (4.886060203452688) + | > loss: 1.1881293058395386 (1.2365166292994823) + | > current_lr: 5e-06 + | > step_time: 0.3006 (0.3075280897875869) + | > loader_time: 0.0085 (0.009250940242445639) + + + --> TIME: 2024-10-26 09:44:37 -- STEP: 12500/30063 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.04377176985144615 (0.05994089479774223) + | > loss_mel_ce: 4.8808465003967285 (4.884314359798452) + | > loss: 1.2311545610427856 (1.2360638137006719) + | > current_lr: 5e-06 + | > step_time: 0.2081 (0.30759490468978906) + | > loader_time: 0.009 (0.009250369892120375) + + + --> TIME: 2024-10-26 09:45:02 -- STEP: 12550/30063 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.04716948792338371 (0.0598782557771499) + | > loss_mel_ce: 4.0573201179504395 (4.882644639110207) + | > loss: 1.0261224508285522 (1.2356307237604176) + | > current_lr: 5e-06 + | > step_time: 0.3625 (0.30754963441673994) + | > loader_time: 0.0093 (0.009248658309419809) + + + --> TIME: 2024-10-26 09:45:27 -- STEP: 12600/30063 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.0436917282640934 (0.05981297798602577) + | > loss_mel_ce: 4.382955074310303 (4.880552474127899) + | > loss: 1.1066616773605347 (1.235091363073337) + | > current_lr: 5e-06 + | > step_time: 0.2528 (0.30748113743842603) + | > loader_time: 0.0084 (0.009247208701239705) + + + --> TIME: 2024-10-26 09:45:52 -- STEP: 12650/30063 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.03802647069096565 (0.05974712433521678) + | > loss_mel_ce: 3.9960849285125732 (4.878706712722799) + | > loss: 1.0085278749465942 (1.2346134593269995) + | > current_lr: 5e-06 + | > step_time: 0.3047 (0.3074466373420984) + | > loader_time: 0.0089 (0.009247009235879658) + + + --> TIME: 2024-10-26 09:46:17 -- STEP: 12700/30063 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.04801427572965622 (0.059683691560635285) + | > loss_mel_ce: 4.893192291259766 (4.8768257044619645) + | > loss: 1.2353016138076782 (1.2341273490368811) + | > current_lr: 5e-06 + | > step_time: 0.3458 (0.3074071559192625) + | > loader_time: 0.0097 (0.009245572371745689) + + + --> TIME: 2024-10-26 09:46:43 -- STEP: 12750/30063 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.04432092607021332 (0.059622032502994686) + | > loss_mel_ce: 4.735030174255371 (4.875061599881058) + | > loss: 1.1948378086090088 (1.2336709081144854) + | > current_lr: 5e-06 + | > step_time: 0.302 (0.30740584324855397) + | > loader_time: 0.0096 (0.00924455132203945) + + + --> TIME: 2024-10-26 09:47:10 -- STEP: 12800/30063 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.04382703825831413 (0.05955506492464324) + | > loss_mel_ce: 4.636072158813477 (4.8732372653111975) + | > loss: 1.1699748039245605 (1.23319808259606) + | > current_lr: 5e-06 + | > step_time: 0.2484 (0.3074237504228949) + | > loader_time: 0.0085 (0.00924404272809626) + + + --> TIME: 2024-10-26 09:47:37 -- STEP: 12850/30063 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.042145732790231705 (0.059492391728531394) + | > loss_mel_ce: 4.935103893280029 (4.8718144065767985) + | > loss: 1.2443124055862427 (1.2328266996214794) + | > current_lr: 5e-06 + | > step_time: 0.357 (0.3074965721902213) + | > loader_time: 0.0088 (0.009242973197758901) + + + --> TIME: 2024-10-26 09:48:03 -- STEP: 12900/30063 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.04121166095137596 (0.05942995931346736) + | > loss_mel_ce: 4.287423133850098 (4.8699677766755665) + | > loss: 1.0821586847305298 (1.2323494340329175) + | > current_lr: 5e-06 + | > step_time: 0.2094 (0.30754117572030326) + | > loader_time: 0.0086 (0.009243348679801299) + + + --> TIME: 2024-10-26 09:48:28 -- STEP: 12950/30063 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.043676894158124924 (0.059366723780581256) + | > loss_mel_ce: 4.128334045410156 (4.868060077387406) + | > loss: 1.043002724647522 (1.2318567003200385) + | > current_lr: 5e-06 + | > step_time: 0.3441 (0.30747406703624913) + | > loader_time: 0.0092 (0.009241971675033283) + + + --> TIME: 2024-10-26 09:48:54 -- STEP: 13000/30063 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.04051762819290161 (0.05930487494399901) + | > loss_mel_ce: 4.365699768066406 (4.866051356774119) + | > loss: 1.1015543937683105 (1.2313390579544556) + | > current_lr: 5e-06 + | > step_time: 0.288 (0.3074841856222882) + | > loader_time: 0.0084 (0.009240871612842268) + + + --> TIME: 2024-10-26 09:49:21 -- STEP: 13050/30063 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.04123494029045105 (0.05924402801155802) + | > loss_mel_ce: 4.169182777404785 (4.864193515448742) + | > loss: 1.0526044368743896 (1.230859385884125) + | > current_lr: 5e-06 + | > step_time: 0.3076 (0.30753996158468266) + | > loader_time: 0.0087 (0.009240716188803486) + + + --> TIME: 2024-10-26 09:49:46 -- STEP: 13100/30063 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.03467089682817459 (0.0591848131260684) + | > loss_mel_ce: 4.013933181762695 (4.862415088515247) + | > loss: 1.0121510028839111 (1.230399975435422) + | > current_lr: 5e-06 + | > step_time: 0.2244 (0.30751952515303593) + | > loader_time: 0.0086 (0.009239344924460838) + + + --> TIME: 2024-10-26 09:50:13 -- STEP: 13150/30063 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.03786170855164528 (0.05912429289616093) + | > loss_mel_ce: 4.219997882843018 (4.860934923918981) + | > loss: 1.0644649267196655 (1.2300148042481183) + | > current_lr: 5e-06 + | > step_time: 0.2625 (0.3075352037814177) + | > loader_time: 0.0086 (0.009238131780588126) + + + --> TIME: 2024-10-26 09:50:37 -- STEP: 13200/30063 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.05002226307988167 (0.059061273262186804) + | > loss_mel_ce: 4.74195671081543 (4.859009072347115) + | > loss: 1.1979947090148926 (1.2295175864795822) + | > current_lr: 5e-06 + | > step_time: 0.2514 (0.3074969845287722) + | > loader_time: 0.009 (0.009237328713590448) + + + --> TIME: 2024-10-26 09:51:03 -- STEP: 13250/30063 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.043223559856414795 (0.05900275300190121) + | > loss_mel_ce: 4.613772869110107 (4.857439465900627) + | > loss: 1.164249062538147 (1.2291105548075885) + | > current_lr: 5e-06 + | > step_time: 0.3609 (0.3074571626591225) + | > loader_time: 0.0094 (0.009236763630273205) + + + --> TIME: 2024-10-26 09:51:27 -- STEP: 13300/30063 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.0421791598200798 (0.05894330901927057) + | > loss_mel_ce: 4.332854270935059 (4.855741078889451) + | > loss: 1.0937583446502686 (1.2286710970742345) + | > current_lr: 5e-06 + | > step_time: 0.2006 (0.3073946033384561) + | > loader_time: 0.0101 (0.009236229559532677) + + + --> TIME: 2024-10-26 09:51:52 -- STEP: 13350/30063 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.041880011558532715 (0.058883763881825646) + | > loss_mel_ce: 4.741331100463867 (4.854050355118304) + | > loss: 1.1958028078079224 (1.2282335298740008) + | > current_lr: 5e-06 + | > step_time: 0.4017 (0.3073310096612129) + | > loader_time: 0.0097 (0.009235905493689833) + + + --> TIME: 2024-10-26 09:52:18 -- STEP: 13400/30063 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.04548204317688942 (0.05882664655015527) + | > loss_mel_ce: 4.100371360778809 (4.852021604562865) + | > loss: 1.0364633798599243 (1.2277120629219858) + | > current_lr: 5e-06 + | > step_time: 0.3597 (0.30735853084877274) + | > loader_time: 0.0093 (0.00923471872486285) + + + --> TIME: 2024-10-26 09:52:44 -- STEP: 13450/30063 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.045175231993198395 (0.05876886596605446) + | > loss_mel_ce: 4.088685989379883 (4.850362415792318) + | > loss: 1.0334652662277222 (1.2272828205810595) + | > current_lr: 5e-06 + | > step_time: 0.3763 (0.3073411065849665) + | > loader_time: 0.0089 (0.009233767871076722) + + + --> TIME: 2024-10-26 09:53:09 -- STEP: 13500/30063 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.04127155616879463 (0.05871015053711543) + | > loss_mel_ce: 4.1588616371154785 (4.848365241297957) + | > loss: 1.0500333309173584 (1.2267688481012973) + | > current_lr: 5e-06 + | > step_time: 0.2473 (0.3073092738610719) + | > loader_time: 0.0098 (0.009233747694227417) + + + --> TIME: 2024-10-26 09:53:34 -- STEP: 13550/30063 -- GLOBAL_STEP: 13550 + | > loss_text_ce: 0.04087664186954498 (0.05865213444339601) + | > loss_mel_ce: 4.036096572875977 (4.846633702908042) + | > loss: 1.0192433595657349 (1.2263214595115484) + | > current_lr: 5e-06 + | > step_time: 0.4447 (0.3072902928947075) + | > loader_time: 0.0112 (0.009232732202733995) + + + --> TIME: 2024-10-26 09:54:00 -- STEP: 13600/30063 -- GLOBAL_STEP: 13600 + | > loss_text_ce: 0.04948098957538605 (0.05859549955218361) + | > loss_mel_ce: 5.147891521453857 (4.845229378710762) + | > loss: 1.2993431091308594 (1.2259562197152298) + | > current_lr: 5e-06 + | > step_time: 0.3242 (0.3072888804533894) + | > loader_time: 0.0097 (0.009232706112020131) + + + --> TIME: 2024-10-26 09:54:26 -- STEP: 13650/30063 -- GLOBAL_STEP: 13650 + | > loss_text_ce: 0.03701704740524292 (0.05853785963367399) + | > loss_mel_ce: 4.636731147766113 (4.843632612175981) + | > loss: 1.1684370040893555 (1.2255426181061355) + | > current_lr: 5e-06 + | > step_time: 0.2117 (0.30730121244004277) + | > loader_time: 0.0092 (0.009231516223250701) + + + --> TIME: 2024-10-26 09:54:52 -- STEP: 13700/30063 -- GLOBAL_STEP: 13700 + | > loss_text_ce: 0.043017659336328506 (0.05848347646231839) + | > loss_mel_ce: 4.437896728515625 (4.841885631989376) + | > loss: 1.12022864818573 (1.2250922772397081) + | > current_lr: 5e-06 + | > step_time: 0.2886 (0.30731421388848773) + | > loader_time: 0.0089 (0.009230976783446124) + + + --> TIME: 2024-10-26 09:55:18 -- STEP: 13750/30063 -- GLOBAL_STEP: 13750 + | > loss_text_ce: 0.042664822190999985 (0.058423461503332184) + | > loss_mel_ce: 4.523960113525391 (4.839983669315672) + | > loss: 1.1416562795639038 (1.2246017828334455) + | > current_lr: 5e-06 + | > step_time: 0.4673 (0.3073263221393924) + | > loader_time: 0.0093 (0.009229874264110195) + + + --> TIME: 2024-10-26 09:55:45 -- STEP: 13800/30063 -- GLOBAL_STEP: 13800 + | > loss_text_ce: 0.047376859933137894 (0.058364842705752415) + | > loss_mel_ce: 5.001344203948975 (4.838097296503994) + | > loss: 1.262180209159851 (1.2241155349171673) + | > current_lr: 5e-06 + | > step_time: 0.3252 (0.3073434847334149) + | > loader_time: 0.0112 (0.009230145630629141) + + + --> TIME: 2024-10-26 09:56:11 -- STEP: 13850/30063 -- GLOBAL_STEP: 13850 + | > loss_text_ce: 0.044620346277952194 (0.058311647763105816) + | > loss_mel_ce: 4.8541388511657715 (4.836692869430832) + | > loss: 1.2246898412704468 (1.2237511294284014) + | > current_lr: 5e-06 + | > step_time: 0.3578 (0.3073566392526722) + | > loader_time: 0.0089 (0.009229391679867057) + + + --> TIME: 2024-10-26 09:56:35 -- STEP: 13900/30063 -- GLOBAL_STEP: 13900 + | > loss_text_ce: 0.040659502148628235 (0.058257095829295474) + | > loss_mel_ce: 4.195016860961914 (4.834990588915422) + | > loss: 1.058919072151184 (1.2233119213023624) + | > current_lr: 5e-06 + | > step_time: 0.2316 (0.30730354104968205) + | > loader_time: 0.0089 (0.009228317240159266) + + + --> TIME: 2024-10-26 09:57:02 -- STEP: 13950/30063 -- GLOBAL_STEP: 13950 + | > loss_text_ce: 0.049711406230926514 (0.05820385539403514) + | > loss_mel_ce: 4.340839385986328 (4.833771834048734) + | > loss: 1.09763765335083 (1.2229939224771267) + | > current_lr: 5e-06 + | > step_time: 0.398 (0.30734325020971187) + | > loader_time: 0.0096 (0.009227300869521253) + + + --> TIME: 2024-10-26 09:57:28 -- STEP: 14000/30063 -- GLOBAL_STEP: 14000 + | > loss_text_ce: 0.039197709411382675 (0.05814947410140706) + | > loss_mel_ce: 3.950216770172119 (4.832381835818278) + | > loss: 0.9973536133766174 (1.2226328276055192) + | > current_lr: 5e-06 + | > step_time: 0.243 (0.30736607020241796) + | > loader_time: 0.0095 (0.009226345351764111) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_14000.pth + + --> TIME: 2024-10-26 09:58:08 -- STEP: 14050/30063 -- GLOBAL_STEP: 14050 + | > loss_text_ce: 0.0439474917948246 (0.0580960465885459) + | > loss_mel_ce: 3.862816572189331 (4.8305715925413395) + | > loss: 0.9766910076141357 (1.2221669099296957) + | > current_lr: 5e-06 + | > step_time: 0.2689 (0.30736765941266836) + | > loader_time: 0.009 (0.009226475304980273) + + + --> TIME: 2024-10-26 09:58:34 -- STEP: 14100/30063 -- GLOBAL_STEP: 14100 + | > loss_text_ce: 0.04417821392416954 (0.058045504410216026) + | > loss_mel_ce: 4.249610900878906 (4.828725071764994) + | > loss: 1.0734472274780273 (1.2216926442308622) + | > current_lr: 5e-06 + | > step_time: 0.3037 (0.30735278844833297) + | > loader_time: 0.0088 (0.009225653807322163) + + + --> TIME: 2024-10-26 09:58:58 -- STEP: 14150/30063 -- GLOBAL_STEP: 14150 + | > loss_text_ce: 0.04187445715069771 (0.057992581342850354) + | > loss_mel_ce: 4.215877056121826 (4.826877176197161) + | > loss: 1.0644378662109375 (1.2212174395790363) + | > current_lr: 5e-06 + | > step_time: 0.302 (0.30730186063072207) + | > loader_time: 0.0084 (0.009224979801649739) + + + --> TIME: 2024-10-26 09:59:23 -- STEP: 14200/30063 -- GLOBAL_STEP: 14200 + | > loss_text_ce: 0.044174693524837494 (0.0579418930065043) + | > loss_mel_ce: 4.8073506355285645 (4.825386373761662) + | > loss: 1.212881326675415 (1.2208320668717498) + | > current_lr: 5e-06 + | > step_time: 0.3479 (0.30725603123785666) + | > loader_time: 0.0099 (0.009224662377800718) + + + --> TIME: 2024-10-26 09:59:50 -- STEP: 14250/30063 -- GLOBAL_STEP: 14250 + | > loss_text_ce: 0.04539482668042183 (0.057888460299686265) + | > loss_mel_ce: 4.5705952644348145 (4.823584514902337) + | > loss: 1.153997540473938 (1.2203682440038306) + | > current_lr: 5e-06 + | > step_time: 0.3375 (0.30727939169030366) + | > loader_time: 0.0086 (0.009225165283470801) + + + --> TIME: 2024-10-26 10:00:15 -- STEP: 14300/30063 -- GLOBAL_STEP: 14300 + | > loss_text_ce: 0.04202690348029137 (0.05783264448254038) + | > loss_mel_ce: 4.775210857391357 (4.821592292418835) + | > loss: 1.2043094635009766 (1.2198562344304325) + | > current_lr: 5e-06 + | > step_time: 0.2728 (0.3072528399787575) + | > loader_time: 0.0087 (0.009224130056954742) + + + --> TIME: 2024-10-26 10:00:40 -- STEP: 14350/30063 -- GLOBAL_STEP: 14350 + | > loss_text_ce: 0.04136889800429344 (0.057778874847531615) + | > loss_mel_ce: 4.076894283294678 (4.8201054398274055) + | > loss: 1.0295658111572266 (1.219471078899263) + | > current_lr: 5e-06 + | > step_time: 0.2637 (0.3072275838951608) + | > loader_time: 0.0083 (0.009222847759100596) + + + --> TIME: 2024-10-26 10:01:05 -- STEP: 14400/30063 -- GLOBAL_STEP: 14400 + | > loss_text_ce: 0.041974857449531555 (0.05772288659044219) + | > loss_mel_ce: 4.497849464416504 (4.818337837424531) + | > loss: 1.1349561214447021 (1.2190151812260341) + | > current_lr: 5e-06 + | > step_time: 0.2724 (0.30719146516587925) + | > loader_time: 0.0091 (0.009222031086683251) + + + --> TIME: 2024-10-26 10:01:30 -- STEP: 14450/30063 -- GLOBAL_STEP: 14450 + | > loss_text_ce: 0.038760215044021606 (0.05766856144404729) + | > loss_mel_ce: 4.33547830581665 (4.816477950469008) + | > loss: 1.0935596227645874 (1.2185366282042327) + | > current_lr: 5e-06 + | > step_time: 0.2294 (0.30714164737186556) + | > loader_time: 0.0082 (0.009221284051254938) + + + --> TIME: 2024-10-26 10:01:56 -- STEP: 14500/30063 -- GLOBAL_STEP: 14500 + | > loss_text_ce: 0.0442153736948967 (0.057619747874808606) + | > loss_mel_ce: 3.9625165462493896 (4.815100455086795) + | > loss: 1.001682996749878 (1.2181800509855658) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.30715769040995555) + | > loader_time: 0.0082 (0.009219908928049) + + + --> TIME: 2024-10-26 10:02:23 -- STEP: 14550/30063 -- GLOBAL_STEP: 14550 + | > loss_text_ce: 0.041927270591259 (0.05756679781190911) + | > loss_mel_ce: 3.902470111846924 (4.813365619993689) + | > loss: 0.986099362373352 (1.2177331047041713) + | > current_lr: 5e-06 + | > step_time: 0.3383 (0.30719332216531475) + | > loader_time: 0.0092 (0.009219135107453315) + + + --> TIME: 2024-10-26 10:02:48 -- STEP: 14600/30063 -- GLOBAL_STEP: 14600 + | > loss_text_ce: 0.04428572580218315 (0.05751727456558637) + | > loss_mel_ce: 4.564116477966309 (4.811764474241689) + | > loss: 1.1521005630493164 (1.2173204374476647) + | > current_lr: 5e-06 + | > step_time: 0.3592 (0.3071636829800794) + | > loader_time: 0.0097 (0.009219111890008983) + + + --> TIME: 2024-10-26 10:03:13 -- STEP: 14650/30063 -- GLOBAL_STEP: 14650 + | > loss_text_ce: 0.03969542309641838 (0.057468018506624884) + | > loss_mel_ce: 4.051859378814697 (4.8105880610039495) + | > loss: 1.0228886604309082 (1.2170140201199169) + | > current_lr: 5e-06 + | > step_time: 0.2336 (0.3071296087466809) + | > loader_time: 0.0088 (0.009217660679345223) + + + --> TIME: 2024-10-26 10:03:40 -- STEP: 14700/30063 -- GLOBAL_STEP: 14700 + | > loss_text_ce: 0.04381052032113075 (0.05741681098836608) + | > loss_mel_ce: 3.815000534057617 (4.8089970258141745) + | > loss: 0.9647027850151062 (1.2166034594320108) + | > current_lr: 5e-06 + | > step_time: 0.2095 (0.3071716681467425) + | > loader_time: 0.0088 (0.009217097467305688) + + + --> TIME: 2024-10-26 10:04:05 -- STEP: 14750/30063 -- GLOBAL_STEP: 14750 + | > loss_text_ce: 0.04406379908323288 (0.057365661565782544) + | > loss_mel_ce: 3.791686534881592 (4.807168190018594) + | > loss: 0.9589375853538513 (1.2161334631281377) + | > current_lr: 5e-06 + | > step_time: 0.2323 (0.30714761995865053) + | > loader_time: 0.0086 (0.00921638509782692) + + + --> TIME: 2024-10-26 10:04:31 -- STEP: 14800/30063 -- GLOBAL_STEP: 14800 + | > loss_text_ce: 0.04454737529158592 (0.05731797180667105) + | > loss_mel_ce: 4.419222354888916 (4.805535913738033) + | > loss: 1.1159424781799316 (1.2157134716212743) + | > current_lr: 5e-06 + | > step_time: 0.1883 (0.30714284146154175) + | > loader_time: 0.0084 (0.009215343820082154) + + + --> TIME: 2024-10-26 10:04:56 -- STEP: 14850/30063 -- GLOBAL_STEP: 14850 + | > loss_text_ce: 0.04080287739634514 (0.057269140851467434) + | > loss_mel_ce: 4.3369011878967285 (4.804428064285292) + | > loss: 1.0944260358810425 (1.215424301519939) + | > current_lr: 5e-06 + | > step_time: 0.3573 (0.30711796161702914) + | > loader_time: 0.0095 (0.009214752161944333) + + + --> TIME: 2024-10-26 10:05:23 -- STEP: 14900/30063 -- GLOBAL_STEP: 14900 + | > loss_text_ce: 0.04454677924513817 (0.057218162164391975) + | > loss_mel_ce: 4.551098823547363 (4.802946955773643) + | > loss: 1.1489113569259644 (1.2150412797087784) + | > current_lr: 5e-06 + | > step_time: 0.2084 (0.3071756732703848) + | > loader_time: 0.0093 (0.009214156426039297) + + + --> TIME: 2024-10-26 10:05:48 -- STEP: 14950/30063 -- GLOBAL_STEP: 14950 + | > loss_text_ce: 0.04580620676279068 (0.057167346125661614) + | > loss_mel_ce: 4.140129089355469 (4.801518270323496) + | > loss: 1.0464838743209839 (1.2146714043378026) + | > current_lr: 5e-06 + | > step_time: 0.3289 (0.3071339155599019) + | > loader_time: 0.009 (0.009213535618223852) + + + --> TIME: 2024-10-26 10:06:14 -- STEP: 15000/30063 -- GLOBAL_STEP: 15000 + | > loss_text_ce: 0.040005140006542206 (0.057116303834070646) + | > loss_mel_ce: 4.481742858886719 (4.8000050004164265) + | > loss: 1.1304370164871216 (1.2142803263068191) + | > current_lr: 5e-06 + | > step_time: 0.3459 (0.3071283787091566) + | > loader_time: 0.0095 (0.009213288338979064) + + + --> TIME: 2024-10-26 10:06:40 -- STEP: 15050/30063 -- GLOBAL_STEP: 15050 + | > loss_text_ce: 0.040979478508234024 (0.05706607895510328) + | > loss_mel_ce: 4.947078704833984 (4.798434780824213) + | > loss: 1.2470145225524902 (1.213875215192173) + | > current_lr: 5e-06 + | > step_time: 0.5219 (0.30713194360764645) + | > loader_time: 0.0098 (0.009212160268891273) + + + --> TIME: 2024-10-26 10:07:05 -- STEP: 15100/30063 -- GLOBAL_STEP: 15100 + | > loss_text_ce: 0.03658517450094223 (0.057017292514295256) + | > loss_mel_ce: 3.9633615016937256 (4.796549914751609) + | > loss: 0.9999866485595703 (1.2133918020504193) + | > current_lr: 5e-06 + | > step_time: 0.2021 (0.30710806890828646) + | > loader_time: 0.0081 (0.00921161286878268) + + + --> TIME: 2024-10-26 10:07:32 -- STEP: 15150/30063 -- GLOBAL_STEP: 15150 + | > loss_text_ce: 0.040347617119550705 (0.05696849400503396) + | > loss_mel_ce: 4.799482822418213 (4.795164297862403) + | > loss: 1.2099575996398926 (1.2130331981772235) + | > current_lr: 5e-06 + | > step_time: 0.2545 (0.3071371274576715) + | > loader_time: 0.0084 (0.009211009390676749) + + + --> TIME: 2024-10-26 10:07:57 -- STEP: 15200/30063 -- GLOBAL_STEP: 15200 + | > loss_text_ce: 0.04689180850982666 (0.05692199065178426) + | > loss_mel_ce: 4.483241558074951 (4.793462210407371) + | > loss: 1.132533311843872 (1.212596050500869) + | > current_lr: 5e-06 + | > step_time: 0.3748 (0.30712282279604286) + | > loader_time: 0.01 (0.009210360552135246) + + + --> TIME: 2024-10-26 10:08:24 -- STEP: 15250/30063 -- GLOBAL_STEP: 15250 + | > loss_text_ce: 0.038457710295915604 (0.05687278647256668) + | > loss_mel_ce: 3.7906277179718018 (4.791401154580652) + | > loss: 0.9572713375091553 (1.2120684854945192) + | > current_lr: 5e-06 + | > step_time: 0.255 (0.30716123544974333) + | > loader_time: 0.0091 (0.00920964497425515) + + + --> TIME: 2024-10-26 10:08:50 -- STEP: 15300/30063 -- GLOBAL_STEP: 15300 + | > loss_text_ce: 0.046418625861406326 (0.056826051765066624) + | > loss_mel_ce: 4.128140926361084 (4.78959283351897) + | > loss: 1.043639898300171 (1.2116047215422767) + | > current_lr: 5e-06 + | > step_time: 0.2714 (0.3071522229948847) + | > loader_time: 0.01 (0.009210147203183621) + + + --> TIME: 2024-10-26 10:09:15 -- STEP: 15350/30063 -- GLOBAL_STEP: 15350 + | > loss_text_ce: 0.036800842732191086 (0.05677706609719728) + | > loss_mel_ce: 4.052865028381348 (4.788072031437369) + | > loss: 1.0224164724349976 (1.211212274612743) + | > current_lr: 5e-06 + | > step_time: 0.2594 (0.3071468064994681) + | > loader_time: 0.0084 (0.009210738794034758) + + + --> TIME: 2024-10-26 10:09:42 -- STEP: 15400/30063 -- GLOBAL_STEP: 15400 + | > loss_text_ce: 0.047413863241672516 (0.056729931444074705) + | > loss_mel_ce: 4.3706769943237305 (4.786643409140687) + | > loss: 1.104522705078125 (1.2108433353707382) + | > current_lr: 5e-06 + | > step_time: 0.1744 (0.307163933856146) + | > loader_time: 0.0092 (0.009210683246711612) + + + --> TIME: 2024-10-26 10:10:09 -- STEP: 15450/30063 -- GLOBAL_STEP: 15450 + | > loss_text_ce: 0.052413519471883774 (0.05668340838963342) + | > loss_mel_ce: 4.598607063293457 (4.7851808600903984) + | > loss: 1.1627551317214966 (1.2104660673357515) + | > current_lr: 5e-06 + | > step_time: 0.3778 (0.30723696580596654) + | > loader_time: 0.0087 (0.009210292992082593) + + + --> TIME: 2024-10-26 10:10:35 -- STEP: 15500/30063 -- GLOBAL_STEP: 15500 + | > loss_text_ce: 0.04548536613583565 (0.056636127450773684) + | > loss_mel_ce: 4.624387264251709 (4.783980648948289) + | > loss: 1.1674681901931763 (1.2101541943050194) + | > current_lr: 5e-06 + | > step_time: 0.2097 (0.3072244505882256) + | > loader_time: 0.0087 (0.009209642456423832) + + + --> TIME: 2024-10-26 10:11:01 -- STEP: 15550/30063 -- GLOBAL_STEP: 15550 + | > loss_text_ce: 0.044909313321113586 (0.05659098759389373) + | > loss_mel_ce: 4.497960090637207 (4.782339907735095) + | > loss: 1.1357173919677734 (1.2097327240295341) + | > current_lr: 5e-06 + | > step_time: 0.2695 (0.3072233274588624) + | > loader_time: 0.009 (0.009209625897300185) + + + --> TIME: 2024-10-26 10:11:27 -- STEP: 15600/30063 -- GLOBAL_STEP: 15600 + | > loss_text_ce: 0.039099402725696564 (0.056543005934605886) + | > loss_mel_ce: 5.13686466217041 (4.780542155152701) + | > loss: 1.293990969657898 (1.2092712904703915) + | > current_lr: 5e-06 + | > step_time: 0.2529 (0.3072501161006774) + | > loader_time: 0.0087 (0.009208962886761375) + + + --> TIME: 2024-10-26 10:11:52 -- STEP: 15650/30063 -- GLOBAL_STEP: 15650 + | > loss_text_ce: 0.040448885411024094 (0.05649542203655057) + | > loss_mel_ce: 4.7154340744018555 (4.7787382614574385) + | > loss: 1.188970685005188 (1.208808421064108) + | > current_lr: 5e-06 + | > step_time: 0.4368 (0.3071940990911117) + | > loader_time: 0.0097 (0.009208658952682525) + + + --> TIME: 2024-10-26 10:12:18 -- STEP: 15700/30063 -- GLOBAL_STEP: 15700 + | > loss_text_ce: 0.0488094799220562 (0.056451956418574124) + | > loss_mel_ce: 4.462857246398926 (4.777311706937791) + | > loss: 1.127916693687439 (1.2084409160333067) + | > current_lr: 5e-06 + | > step_time: 0.2094 (0.3072000407717024) + | > loader_time: 0.0082 (0.009208221602591719) + + + --> TIME: 2024-10-26 10:12:44 -- STEP: 15750/30063 -- GLOBAL_STEP: 15750 + | > loss_text_ce: 0.04876994341611862 (0.056408973392512954) + | > loss_mel_ce: 4.465559482574463 (4.775703950775983) + | > loss: 1.1285823583602905 (1.208028231227208) + | > current_lr: 5e-06 + | > step_time: 0.3366 (0.30719158611600295) + | > loader_time: 0.0084 (0.009207584214588933) + + + --> TIME: 2024-10-26 10:13:10 -- STEP: 15800/30063 -- GLOBAL_STEP: 15800 + | > loss_text_ce: 0.04530955106019974 (0.05636346678332049) + | > loss_mel_ce: 4.376796245574951 (4.774217996220035) + | > loss: 1.1055264472961426 (1.2076453659428819) + | > current_lr: 5e-06 + | > step_time: 0.2742 (0.3072188647185693) + | > loader_time: 0.0091 (0.00920685861684097) + + + --> TIME: 2024-10-26 10:13:37 -- STEP: 15850/30063 -- GLOBAL_STEP: 15850 + | > loss_text_ce: 0.04494616389274597 (0.05631750497485174) + | > loss_mel_ce: 4.261308670043945 (4.772812532491101) + | > loss: 1.0765637159347534 (1.2072825095555753) + | > current_lr: 5e-06 + | > step_time: 0.2524 (0.30724659160108525) + | > loader_time: 0.0091 (0.00920619138783835) + + + --> TIME: 2024-10-26 10:14:02 -- STEP: 15900/30063 -- GLOBAL_STEP: 15900 + | > loss_text_ce: 0.0409160777926445 (0.05627056751657946) + | > loss_mel_ce: 4.369264125823975 (4.771005953077989) + | > loss: 1.1025450229644775 (1.2068191303284657) + | > current_lr: 5e-06 + | > step_time: 0.2047 (0.3072245821412997) + | > loader_time: 0.0092 (0.00920683652349986) + + + --> TIME: 2024-10-26 10:14:27 -- STEP: 15950/30063 -- GLOBAL_STEP: 15950 + | > loss_text_ce: 0.04240712523460388 (0.05622717815460072) + | > loss_mel_ce: 4.5367255210876465 (4.769575938999082) + | > loss: 1.1447831392288208 (1.2064507794940726) + | > current_lr: 5e-06 + | > step_time: 0.3765 (0.30720732385461913) + | > loader_time: 0.0093 (0.009206429932932102) + + + --> TIME: 2024-10-26 10:14:52 -- STEP: 16000/30063 -- GLOBAL_STEP: 16000 + | > loss_text_ce: 0.03799178823828697 (0.05618241453706276) + | > loss_mel_ce: 3.8519790172576904 (4.767783997043956) + | > loss: 0.9724926948547363 (1.2059916030764573) + | > current_lr: 5e-06 + | > step_time: 0.2501 (0.30717216457426477) + | > loader_time: 0.0089 (0.009206162214279156) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_16000.pth + + --> TIME: 2024-10-26 10:15:29 -- STEP: 16050/30063 -- GLOBAL_STEP: 16050 + | > loss_text_ce: 0.0433027409017086 (0.056138138589782775) + | > loss_mel_ce: 3.5416133403778076 (4.7660342976163) + | > loss: 0.8962290287017822 (1.2055431092268203) + | > current_lr: 5e-06 + | > step_time: 0.2751 (0.3071707897245691) + | > loader_time: 0.0092 (0.009206670674950887) + + + --> TIME: 2024-10-26 10:15:55 -- STEP: 16100/30063 -- GLOBAL_STEP: 16100 + | > loss_text_ce: 0.042961958795785904 (0.05609324514680757) + | > loss_mel_ce: 4.808752059936523 (4.764452935790413) + | > loss: 1.2129285335540771 (1.2051365454152498) + | > current_lr: 5e-06 + | > step_time: 0.2564 (0.30716367959976126) + | > loader_time: 0.0091 (0.009205960072345596) + + + --> TIME: 2024-10-26 10:16:21 -- STEP: 16150/30063 -- GLOBAL_STEP: 16150 + | > loss_text_ce: 0.038537364453077316 (0.05604989977513109) + | > loss_mel_ce: 4.123438835144043 (4.763031597506509) + | > loss: 1.0404940843582153 (1.2047703745158458) + | > current_lr: 5e-06 + | > step_time: 0.2584 (0.30717197275014047) + | > loader_time: 0.0088 (0.009205534686983166) + + + --> TIME: 2024-10-26 10:16:46 -- STEP: 16200/30063 -- GLOBAL_STEP: 16200 + | > loss_text_ce: 0.04190023988485336 (0.05600547933238145) + | > loss_mel_ce: 4.167117595672607 (4.7617017441619955) + | > loss: 1.0522544384002686 (1.204426806068714) + | > current_lr: 5e-06 + | > step_time: 0.2715 (0.3071417335080505) + | > loader_time: 0.0088 (0.009204869314476276) + + + --> TIME: 2024-10-26 10:17:13 -- STEP: 16250/30063 -- GLOBAL_STEP: 16250 + | > loss_text_ce: 0.04096363112330437 (0.055963666643316826) + | > loss_mel_ce: 4.232492446899414 (4.760209155038676) + | > loss: 1.0683640241622925 (1.2040432056096877) + | > current_lr: 5e-06 + | > step_time: 0.2873 (0.3071574671085058) + | > loader_time: 0.0095 (0.009204269013038028) + + + --> TIME: 2024-10-26 10:17:39 -- STEP: 16300/30063 -- GLOBAL_STEP: 16300 + | > loss_text_ce: 0.03716060519218445 (0.05591825027795829) + | > loss_mel_ce: 3.8711676597595215 (4.758803826007364) + | > loss: 0.9770820736885071 (1.2036805192714812) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.30717195690775145) + | > loader_time: 0.009 (0.009203778951445953) + + + --> TIME: 2024-10-26 10:18:05 -- STEP: 16350/30063 -- GLOBAL_STEP: 16350 + | > loss_text_ce: 0.03891333192586899 (0.05587286821677587) + | > loss_mel_ce: 4.701501846313477 (4.75744706156785) + | > loss: 1.1851037740707397 (1.2033299826409103) + | > current_lr: 5e-06 + | > step_time: 0.374 (0.3071844811979048) + | > loader_time: 0.009 (0.009203296757619297) + + + --> TIME: 2024-10-26 10:18:30 -- STEP: 16400/30063 -- GLOBAL_STEP: 16400 + | > loss_text_ce: 0.04554561898112297 (0.0558287994188778) + | > loss_mel_ce: 4.241091251373291 (4.75582186351461) + | > loss: 1.0716592073440552 (1.2029126659544496) + | > current_lr: 5e-06 + | > step_time: 0.2587 (0.30715752661228113) + | > loader_time: 0.0089 (0.009202923236823644) + + + --> TIME: 2024-10-26 10:18:56 -- STEP: 16450/30063 -- GLOBAL_STEP: 16450 + | > loss_text_ce: 0.044093504548072815 (0.05578888438913769) + | > loss_mel_ce: 4.322441101074219 (4.754467945171334) + | > loss: 1.091633677482605 (1.2025642076198084) + | > current_lr: 5e-06 + | > step_time: 0.2325 (0.3071497359174356) + | > loader_time: 0.0086 (0.009202700632321417) + + + --> TIME: 2024-10-26 10:19:22 -- STEP: 16500/30063 -- GLOBAL_STEP: 16500 + | > loss_text_ce: 0.04357598349452019 (0.05574395423382511) + | > loss_mel_ce: 3.8082895278930664 (4.752983321637808) + | > loss: 0.9629663825035095 (1.2021818191716158) + | > current_lr: 5e-06 + | > step_time: 0.2344 (0.3071573809132424) + | > loader_time: 0.0092 (0.009202358144702315) + + + --> TIME: 2024-10-26 10:19:48 -- STEP: 16550/30063 -- GLOBAL_STEP: 16550 + | > loss_text_ce: 0.04050175100564957 (0.055700317655670226) + | > loss_mel_ce: 4.3687872886657715 (4.751424171902973) + | > loss: 1.1023222208023071 (1.2017811225836361) + | > current_lr: 5e-06 + | > step_time: 0.2637 (0.30718870956732) + | > loader_time: 0.0091 (0.009201943334135746) + + + --> TIME: 2024-10-26 10:20:14 -- STEP: 16600/30063 -- GLOBAL_STEP: 16600 + | > loss_text_ce: 0.04490629583597183 (0.055658726930079595) + | > loss_mel_ce: 4.52753210067749 (4.750370768840044) + | > loss: 1.1431095600128174 (1.201507374123636) + | > current_lr: 5e-06 + | > step_time: 0.2327 (0.3071810895850853) + | > loader_time: 0.009 (0.009201699251151928) + + + --> TIME: 2024-10-26 10:20:40 -- STEP: 16650/30063 -- GLOBAL_STEP: 16650 + | > loss_text_ce: 0.04312106594443321 (0.05561805011631839) + | > loss_mel_ce: 4.438338756561279 (4.748976845483512) + | > loss: 1.1203649044036865 (1.2011487240762677) + | > current_lr: 5e-06 + | > step_time: 0.2471 (0.30717465711427455) + | > loader_time: 0.0087 (0.009200727545821252) + + + --> TIME: 2024-10-26 10:21:06 -- STEP: 16700/30063 -- GLOBAL_STEP: 16700 + | > loss_text_ce: 0.04048642888665199 (0.0555751031224509) + | > loss_mel_ce: 4.307858943939209 (4.7476146745538905) + | > loss: 1.0870863199234009 (1.2007974446141072) + | > current_lr: 5e-06 + | > step_time: 0.2548 (0.30717023701011) + | > loader_time: 0.009 (0.009200155892058028) + + + --> TIME: 2024-10-26 10:21:31 -- STEP: 16750/30063 -- GLOBAL_STEP: 16750 + | > loss_text_ce: 0.040048204362392426 (0.055533742187183203) + | > loss_mel_ce: 4.694337368011475 (4.745984354175729) + | > loss: 1.1835963726043701 (1.2003795242736583) + | > current_lr: 5e-06 + | > step_time: 0.4174 (0.30714778616890914) + | > loader_time: 0.0111 (0.009199771980741112) + + + --> TIME: 2024-10-26 10:21:57 -- STEP: 16800/30063 -- GLOBAL_STEP: 16800 + | > loss_text_ce: 0.039602652192115784 (0.05548979050812436) + | > loss_mel_ce: 4.281269550323486 (4.744260021774532) + | > loss: 1.0802180767059326 (1.1999374532664102) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.30714156932773984) + | > loader_time: 0.0086 (0.009199361119951502) + + + --> TIME: 2024-10-26 10:22:23 -- STEP: 16850/30063 -- GLOBAL_STEP: 16850 + | > loss_text_ce: 0.046769265085458755 (0.05544806770228659) + | > loss_mel_ce: 3.8877689838409424 (4.742518375068449) + | > loss: 0.9836345911026001 (1.1994916108843123) + | > current_lr: 5e-06 + | > step_time: 0.3291 (0.30716165212919816) + | > loader_time: 0.0095 (0.009199207204031997) + + + --> TIME: 2024-10-26 10:22:49 -- STEP: 16900/30063 -- GLOBAL_STEP: 16900 + | > loss_text_ce: 0.03546395152807236 (0.055405777855693505) + | > loss_mel_ce: 4.316732883453369 (4.741041592324261) + | > loss: 1.0880491733551025 (1.199111842724698) + | > current_lr: 5e-06 + | > step_time: 0.2015 (0.3071435513439962) + | > loader_time: 0.0086 (0.009198224784354468) + + + --> TIME: 2024-10-26 10:23:14 -- STEP: 16950/30063 -- GLOBAL_STEP: 16950 + | > loss_text_ce: 0.039047811180353165 (0.05536647455772283) + | > loss_mel_ce: 3.509335517883301 (4.73972156647031) + | > loss: 0.8870958089828491 (1.198772010458605) + | > current_lr: 5e-06 + | > step_time: 0.2995 (0.3071252404834661) + | > loader_time: 0.0088 (0.009197952669981992) + + + --> TIME: 2024-10-26 10:23:39 -- STEP: 17000/30063 -- GLOBAL_STEP: 17000 + | > loss_text_ce: 0.03945689648389816 (0.05532348956540219) + | > loss_mel_ce: 3.2146244049072266 (4.738134764713387) + | > loss: 0.8135203123092651 (1.198364563777165) + | > current_lr: 5e-06 + | > step_time: 0.3078 (0.3071135703255141) + | > loader_time: 0.01 (0.009197773414499604) + + + --> TIME: 2024-10-26 10:24:05 -- STEP: 17050/30063 -- GLOBAL_STEP: 17050 + | > loss_text_ce: 0.044667892158031464 (0.05528254840140635) + | > loss_mel_ce: 3.9544951915740967 (4.736727894506136) + | > loss: 0.9997907876968384 (1.1980026109407365) + | > current_lr: 5e-06 + | > step_time: 0.503 (0.30712926900981036) + | > loader_time: 0.0102 (0.009197452676610846) + + + --> TIME: 2024-10-26 10:24:31 -- STEP: 17100/30063 -- GLOBAL_STEP: 17100 + | > loss_text_ce: 0.047118328511714935 (0.055242234962986844) + | > loss_mel_ce: 4.508616924285889 (4.735332914834819) + | > loss: 1.138933777809143 (1.197643787655216) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.3071018864118558) + | > loader_time: 0.0099 (0.00919719725324395) + + + --> TIME: 2024-10-26 10:24:57 -- STEP: 17150/30063 -- GLOBAL_STEP: 17150 + | > loss_text_ce: 0.04781264066696167 (0.055202827463605234) + | > loss_mel_ce: 3.8932547569274902 (4.7337684449927) + | > loss: 0.9852668642997742 (1.1972428183027317) + | > current_lr: 5e-06 + | > step_time: 0.2611 (0.3071117809840604) + | > loader_time: 0.0088 (0.009196675233868731) + + + --> TIME: 2024-10-26 10:25:24 -- STEP: 17200/30063 -- GLOBAL_STEP: 17200 + | > loss_text_ce: 0.040523264557123184 (0.05516247089259144) + | > loss_mel_ce: 4.670137882232666 (4.732437047625685) + | > loss: 1.1776652336120605 (1.1968998798312132) + | > current_lr: 5e-06 + | > step_time: 0.2723 (0.3071597906739207) + | > loader_time: 0.0097 (0.009196376301521467) + + + --> TIME: 2024-10-26 10:25:48 -- STEP: 17250/30063 -- GLOBAL_STEP: 17250 + | > loss_text_ce: 0.035356100648641586 (0.05512224782312252) + | > loss_mel_ce: 4.238561630249023 (4.730935570661562) + | > loss: 1.0684794187545776 (1.1965144548312465) + | > current_lr: 5e-06 + | > step_time: 0.2326 (0.30711116070678196) + | > loader_time: 0.0087 (0.009196073393890813) + + + --> TIME: 2024-10-26 10:26:14 -- STEP: 17300/30063 -- GLOBAL_STEP: 17300 + | > loss_text_ce: 0.04317530617117882 (0.05508142309618642) + | > loss_mel_ce: 3.9369611740112305 (4.729399691647866) + | > loss: 0.9950340986251831 (1.196120278890421) + | > current_lr: 5e-06 + | > step_time: 0.3566 (0.30709923429985186) + | > loader_time: 0.01 (0.0091955502736086) + + + --> TIME: 2024-10-26 10:26:40 -- STEP: 17350/30063 -- GLOBAL_STEP: 17350 + | > loss_text_ce: 0.039038512855768204 (0.0550402999720844) + | > loss_mel_ce: 3.181663990020752 (4.727942509513756) + | > loss: 0.8051756024360657 (1.1957457025971796) + | > current_lr: 5e-06 + | > step_time: 0.3064 (0.307100561908754) + | > loader_time: 0.0101 (0.009195734004809455) + + + --> TIME: 2024-10-26 10:27:06 -- STEP: 17400/30063 -- GLOBAL_STEP: 17400 + | > loss_text_ce: 0.03947627544403076 (0.05500068341478184) + | > loss_mel_ce: 4.37392520904541 (4.726390517196428) + | > loss: 1.1033504009246826 (1.1953478003610127) + | > current_lr: 5e-06 + | > step_time: 0.2433 (0.3070966035470186) + | > loader_time: 0.0086 (0.009194830601242755) + + + --> TIME: 2024-10-26 10:27:32 -- STEP: 17450/30063 -- GLOBAL_STEP: 17450 + | > loss_text_ce: 0.041066497564315796 (0.05496199092926465) + | > loss_mel_ce: 3.897260904312134 (4.724871151180862) + | > loss: 0.9845818281173706 (1.1949582857252181) + | > current_lr: 5e-06 + | > step_time: 0.3108 (0.30711090228618987) + | > loader_time: 0.0094 (0.009194785011531956) + + + --> TIME: 2024-10-26 10:27:58 -- STEP: 17500/30063 -- GLOBAL_STEP: 17500 + | > loss_text_ce: 0.03808281943202019 (0.05492267328287864) + | > loss_mel_ce: 4.35214376449585 (4.723537221949434) + | > loss: 1.0975565910339355 (1.1946149740082865) + | > current_lr: 5e-06 + | > step_time: 0.2512 (0.3071255578994743) + | > loader_time: 0.009 (0.009194582584926045) + + + --> TIME: 2024-10-26 10:28:24 -- STEP: 17550/30063 -- GLOBAL_STEP: 17550 + | > loss_text_ce: 0.043960507959127426 (0.05488439960824448) + | > loss_mel_ce: 4.0187859535217285 (4.722077009548824) + | > loss: 1.0156866312026978 (1.1942403524893292) + | > current_lr: 5e-06 + | > step_time: 0.3378 (0.3071235515724892) + | > loader_time: 0.0091 (0.00919423203862289) + + + --> TIME: 2024-10-26 10:28:50 -- STEP: 17600/30063 -- GLOBAL_STEP: 17600 + | > loss_text_ce: 0.038897883147001266 (0.05484520955540398) + | > loss_mel_ce: 4.040969371795654 (4.7205007775398755) + | > loss: 1.0199668407440186 (1.193836496983739) + | > current_lr: 5e-06 + | > step_time: 0.1888 (0.3071462061865755) + | > loader_time: 0.0085 (0.009194205443967457) + + + --> TIME: 2024-10-26 10:29:16 -- STEP: 17650/30063 -- GLOBAL_STEP: 17650 + | > loss_text_ce: 0.04163587838411331 (0.05480678415733241) + | > loss_mel_ce: 4.473209857940674 (4.719046802129009) + | > loss: 1.128711462020874 (1.1934633967761605) + | > current_lr: 5e-06 + | > step_time: 0.4451 (0.30713789226650756) + | > loader_time: 0.0097 (0.009194080633752388) + + + --> TIME: 2024-10-26 10:29:42 -- STEP: 17700/30063 -- GLOBAL_STEP: 17700 + | > loss_text_ce: 0.039814453572034836 (0.054767702346678035) + | > loss_mel_ce: 4.0652008056640625 (4.717437150774694) + | > loss: 1.0262538194656372 (1.19305121347729) + | > current_lr: 5e-06 + | > step_time: 0.3101 (0.3071467101910684) + | > loader_time: 0.0096 (0.009194013385449406) + + + --> TIME: 2024-10-26 10:30:09 -- STEP: 17750/30063 -- GLOBAL_STEP: 17750 + | > loss_text_ce: 0.04518073424696922 (0.05473061914804943) + | > loss_mel_ce: 4.781771659851074 (4.716332380979824) + | > loss: 1.2067381143569946 (1.192765750233555) + | > current_lr: 5e-06 + | > step_time: 0.3357 (0.30718487059902044) + | > loader_time: 0.0082 (0.009193577752986402) + + + --> TIME: 2024-10-26 10:30:35 -- STEP: 17800/30063 -- GLOBAL_STEP: 17800 + | > loss_text_ce: 0.04087599366903305 (0.054692382112647576) + | > loss_mel_ce: 3.6017773151397705 (4.714913690532182) + | > loss: 0.9106633067131042 (1.1924015183428691) + | > current_lr: 5e-06 + | > step_time: 0.2468 (0.30719181855073047) + | > loader_time: 0.0087 (0.009193222549524175) + + + --> TIME: 2024-10-26 10:31:01 -- STEP: 17850/30063 -- GLOBAL_STEP: 17850 + | > loss_text_ce: 0.03765598312020302 (0.054652519798537434) + | > loss_mel_ce: 3.914109706878662 (4.713439393631205) + | > loss: 0.9879414439201355 (1.1920229785348846) + | > current_lr: 5e-06 + | > step_time: 0.3452 (0.3071975786011417) + | > loader_time: 0.0093 (0.009193015258853148) + + + --> TIME: 2024-10-26 10:31:27 -- STEP: 17900/30063 -- GLOBAL_STEP: 17900 + | > loss_text_ce: 0.037318501621484756 (0.05461473889128605) + | > loss_mel_ce: 3.9009342193603516 (4.711951555286693) + | > loss: 0.9845631718635559 (1.1916415737327908) + | > current_lr: 5e-06 + | > step_time: 0.2371 (0.30722389139942535) + | > loader_time: 0.0085 (0.009192481467177719) + + + --> TIME: 2024-10-26 10:31:53 -- STEP: 17950/30063 -- GLOBAL_STEP: 17950 + | > loss_text_ce: 0.04095972329378128 (0.05457684978116338) + | > loss_mel_ce: 4.091490268707275 (4.710444831542637) + | > loss: 1.0331125259399414 (1.1912554205393722) + | > current_lr: 5e-06 + | > step_time: 0.359 (0.30723013289459494) + | > loader_time: 0.009 (0.009192028085501387) + + + --> TIME: 2024-10-26 10:32:19 -- STEP: 18000/30063 -- GLOBAL_STEP: 18000 + | > loss_text_ce: 0.03939161077141762 (0.054537837034919025) + | > loss_mel_ce: 3.916841745376587 (4.709029821938925) + | > loss: 0.9890583157539368 (1.1908919149471646) + | > current_lr: 5e-06 + | > step_time: 0.2091 (0.3072247031794649) + | > loader_time: 0.0089 (0.009191448714998012) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_18000.pth + + --> TIME: 2024-10-26 10:32:55 -- STEP: 18050/30063 -- GLOBAL_STEP: 18050 + | > loss_text_ce: 0.04627082869410515 (0.05450113017328705) + | > loss_mel_ce: 4.558797836303711 (4.707421590205362) + | > loss: 1.151267170906067 (1.190480680284076) + | > current_lr: 5e-06 + | > step_time: 0.3123 (0.30722545897201103) + | > loader_time: 0.0089 (0.009192263188454558) + + + --> TIME: 2024-10-26 10:33:20 -- STEP: 18100/30063 -- GLOBAL_STEP: 18100 + | > loss_text_ce: 0.045661311596632004 (0.0544629276150743) + | > loss_mel_ce: 4.6822099685668945 (4.705614430864858) + | > loss: 1.181967854499817 (1.1900193397873653) + | > current_lr: 5e-06 + | > step_time: 0.4081 (0.3071791991739637) + | > loader_time: 0.0095 (0.009191941793452285) + + + --> TIME: 2024-10-26 10:33:46 -- STEP: 18150/30063 -- GLOBAL_STEP: 18150 + | > loss_text_ce: 0.03763343766331673 (0.054426144896788195) + | > loss_mel_ce: 3.88830828666687 (4.704350131234501) + | > loss: 0.9814854264259338 (1.1896940691937403) + | > current_lr: 5e-06 + | > step_time: 0.312 (0.3071629131201536) + | > loader_time: 0.0112 (0.009191382161185077) + + + --> TIME: 2024-10-26 10:34:12 -- STEP: 18200/30063 -- GLOBAL_STEP: 18200 + | > loss_text_ce: 0.04023650661110878 (0.054387974888700175) + | > loss_mel_ce: 3.9460573196411133 (4.702855506323179) + | > loss: 0.9965734481811523 (1.1893108704522388) + | > current_lr: 5e-06 + | > step_time: 0.2722 (0.30719377386700897) + | > loader_time: 0.0095 (0.009191266154194898) + + + --> TIME: 2024-10-26 10:34:37 -- STEP: 18250/30063 -- GLOBAL_STEP: 18250 + | > loss_text_ce: 0.04055166617035866 (0.05435099944043628) + | > loss_mel_ce: 3.8243253231048584 (4.701484044362402) + | > loss: 0.9662192463874817 (1.1889587611074284) + | > current_lr: 5e-06 + | > step_time: 0.3358 (0.3071709747183803) + | > loader_time: 0.0089 (0.009191069825054824) + + + --> TIME: 2024-10-26 10:35:04 -- STEP: 18300/30063 -- GLOBAL_STEP: 18300 + | > loss_text_ce: 0.03768610954284668 (0.054315838021902894) + | > loss_mel_ce: 4.158621788024902 (4.700495174334982) + | > loss: 1.049077033996582 (1.188702753235078) + | > current_lr: 5e-06 + | > step_time: 0.2248 (0.30720130322409356) + | > loader_time: 0.0091 (0.009190986390973682) + + + --> TIME: 2024-10-26 10:35:31 -- STEP: 18350/30063 -- GLOBAL_STEP: 18350 + | > loss_text_ce: 0.04277918115258217 (0.054279178021632825) + | > loss_mel_ce: 4.227875232696533 (4.699156409531246) + | > loss: 1.067663550376892 (1.1883588970259644) + | > current_lr: 5e-06 + | > step_time: 0.3822 (0.30723853582907) + | > loader_time: 0.0097 (0.009191028987354391) + + + --> TIME: 2024-10-26 10:35:57 -- STEP: 18400/30063 -- GLOBAL_STEP: 18400 + | > loss_text_ce: 0.038747839629650116 (0.05424266141596353) + | > loss_mel_ce: 3.8206591606140137 (4.6976212487402) + | > loss: 0.9648517370223999 (1.1879659776583924) + | > current_lr: 5e-06 + | > step_time: 0.2017 (0.3072556453813666) + | > loader_time: 0.0089 (0.00919165252343466) + + + --> TIME: 2024-10-26 10:36:22 -- STEP: 18450/30063 -- GLOBAL_STEP: 18450 + | > loss_text_ce: 0.03624747321009636 (0.05420499921208454) + | > loss_mel_ce: 4.963409423828125 (4.696255014478979) + | > loss: 1.2499141693115234 (1.187615003547045) + | > current_lr: 5e-06 + | > step_time: 0.336 (0.3072300121519289) + | > loader_time: 0.0092 (0.009191361610805415) + + + --> TIME: 2024-10-26 10:36:48 -- STEP: 18500/30063 -- GLOBAL_STEP: 18500 + | > loss_text_ce: 0.04001897573471069 (0.05416773069166637) + | > loss_mel_ce: 4.327743053436279 (4.694811897277814) + | > loss: 1.0919405221939087 (1.1872449071278415) + | > current_lr: 5e-06 + | > step_time: 0.2456 (0.3072065354940042) + | > loader_time: 0.0098 (0.009191492183788381) + + + --> TIME: 2024-10-26 10:37:16 -- STEP: 18550/30063 -- GLOBAL_STEP: 18550 + | > loss_text_ce: 0.03907039016485214 (0.05413121601641963) + | > loss_mel_ce: 4.789095878601074 (4.693688231851194) + | > loss: 1.207041621208191 (1.1869548621106965) + | > current_lr: 5e-06 + | > step_time: 0.4006 (0.3072780712981094) + | > loader_time: 0.0095 (0.009191907384003573) + + + --> TIME: 2024-10-26 10:37:42 -- STEP: 18600/30063 -- GLOBAL_STEP: 18600 + | > loss_text_ce: 0.04286431148648262 (0.05409700048546621) + | > loss_mel_ce: 4.108419418334961 (4.692285915561883) + | > loss: 1.0378209352493286 (1.1865957291600502) + | > current_lr: 5e-06 + | > step_time: 0.2043 (0.30730634908522175) + | > loader_time: 0.0105 (0.009191869151207692) + + + --> TIME: 2024-10-26 10:38:09 -- STEP: 18650/30063 -- GLOBAL_STEP: 18650 + | > loss_text_ce: 0.04038739949464798 (0.05406224778266583) + | > loss_mel_ce: 4.320259094238281 (4.690964511942906) + | > loss: 1.090161681175232 (1.1862566900700704) + | > current_lr: 5e-06 + | > step_time: 0.4698 (0.3073324466454743) + | > loader_time: 0.01 (0.009191822596593431) + + + --> TIME: 2024-10-26 10:38:34 -- STEP: 18700/30063 -- GLOBAL_STEP: 18700 + | > loss_text_ce: 0.03680761903524399 (0.054026252737736806) + | > loss_mel_ce: 2.8311071395874023 (4.689311041270967) + | > loss: 0.7169786691665649 (1.185834323649098) + | > current_lr: 5e-06 + | > step_time: 0.2176 (0.30730730599897926) + | > loader_time: 0.0082 (0.00919123400978861) + + + --> TIME: 2024-10-26 10:39:00 -- STEP: 18750/30063 -- GLOBAL_STEP: 18750 + | > loss_text_ce: 0.03845355659723282 (0.05399259481549248) + | > loss_mel_ce: 4.2195868492126465 (4.688146561749756) + | > loss: 1.0645101070404053 (1.1855347892634058) + | > current_lr: 5e-06 + | > step_time: 0.3585 (0.30733494853973237) + | > loader_time: 0.0091 (0.009191167424519833) + + + --> TIME: 2024-10-26 10:39:26 -- STEP: 18800/30063 -- GLOBAL_STEP: 18800 + | > loss_text_ce: 0.043420884758234024 (0.05395625832193378) + | > loss_mel_ce: 4.564781665802002 (4.686790459168698) + | > loss: 1.1520506143569946 (1.185186679486897) + | > current_lr: 5e-06 + | > step_time: 0.4159 (0.3073350629147049) + | > loader_time: 0.0101 (0.009190775054566376) + + + --> TIME: 2024-10-26 10:39:52 -- STEP: 18850/30063 -- GLOBAL_STEP: 18850 + | > loss_text_ce: 0.04348982125520706 (0.05391964659924833) + | > loss_mel_ce: 4.379343032836914 (4.68544303458943) + | > loss: 1.1057082414627075 (1.184840670403496) + | > current_lr: 5e-06 + | > step_time: 0.2547 (0.30731246959625425) + | > loader_time: 0.0089 (0.009190998962766582) + + + --> TIME: 2024-10-26 10:40:19 -- STEP: 18900/30063 -- GLOBAL_STEP: 18900 + | > loss_text_ce: 0.04539928957819939 (0.053884341423670636) + | > loss_mel_ce: 4.425279140472412 (4.684283669953606) + | > loss: 1.1176695823669434 (1.1845420029459781) + | > current_lr: 5e-06 + | > step_time: 0.3204 (0.30736202522560263) + | > loader_time: 0.0091 (0.00919123018860183) + + + --> TIME: 2024-10-26 10:40:45 -- STEP: 18950/30063 -- GLOBAL_STEP: 18950 + | > loss_text_ce: 0.041365984827280045 (0.053849544621473094) + | > loss_mel_ce: 4.395998477935791 (4.682991876891527) + | > loss: 1.1093411445617676 (1.1842103554705494) + | > current_lr: 5e-06 + | > step_time: 0.309 (0.3073627997954461) + | > loader_time: 0.0093 (0.009191405678801896) + + + --> TIME: 2024-10-26 10:41:11 -- STEP: 19000/30063 -- GLOBAL_STEP: 19000 + | > loss_text_ce: 0.039157744497060776 (0.0538127648673558) + | > loss_mel_ce: 4.922301769256592 (4.681818575030859) + | > loss: 1.240364909172058 (1.1839078350851386) + | > current_lr: 5e-06 + | > step_time: 0.2482 (0.3073559956048648) + | > loader_time: 0.0086 (0.009191350623180963) + + + --> TIME: 2024-10-26 10:41:36 -- STEP: 19050/30063 -- GLOBAL_STEP: 19050 + | > loss_text_ce: 0.04159564897418022 (0.05378115444514992) + | > loss_mel_ce: 4.3953070640563965 (4.680683386294529) + | > loss: 1.1092256307601929 (1.1836161352611876) + | > current_lr: 5e-06 + | > step_time: 0.2742 (0.30734349662550337) + | > loader_time: 0.0087 (0.009191053095139205) + + + --> TIME: 2024-10-26 10:42:01 -- STEP: 19100/30063 -- GLOBAL_STEP: 19100 + | > loss_text_ce: 0.039685122668743134 (0.05374634405432244) + | > loss_mel_ce: 4.304113864898682 (4.679344863429723) + | > loss: 1.0859497785568237 (1.1832728019434724) + | > current_lr: 5e-06 + | > step_time: 0.3324 (0.3073229928790573) + | > loader_time: 0.0096 (0.009190451636988428) + + + --> TIME: 2024-10-26 10:42:28 -- STEP: 19150/30063 -- GLOBAL_STEP: 19150 + | > loss_text_ce: 0.03240034356713295 (0.05371191226237732) + | > loss_mel_ce: 3.9657344818115234 (4.677964345294544) + | > loss: 0.9995337128639221 (1.1829190644489544) + | > current_lr: 5e-06 + | > step_time: 0.2881 (0.3073424306675275) + | > loader_time: 0.0089 (0.00918986044104351) + + + --> TIME: 2024-10-26 10:42:54 -- STEP: 19200/30063 -- GLOBAL_STEP: 19200 + | > loss_text_ce: 0.04475318640470505 (0.05367737323103923) + | > loss_mel_ce: 4.296286582946777 (4.676690604661883) + | > loss: 1.0852599143981934 (1.1825919945320718) + | > current_lr: 5e-06 + | > step_time: 0.2713 (0.3073449017604182) + | > loader_time: 0.0086 (0.00918955376992619) + + + --> TIME: 2024-10-26 10:43:19 -- STEP: 19250/30063 -- GLOBAL_STEP: 19250 + | > loss_text_ce: 0.04122244194149971 (0.053643118428719355) + | > loss_mel_ce: 3.654003858566284 (4.675108015766375) + | > loss: 0.9238065481185913 (1.182187783606639) + | > current_lr: 5e-06 + | > step_time: 0.339 (0.30733390674343264) + | > loader_time: 0.0087 (0.009189260780037182) + + + --> TIME: 2024-10-26 10:43:44 -- STEP: 19300/30063 -- GLOBAL_STEP: 19300 + | > loss_text_ce: 0.04265483468770981 (0.05360711608680403) + | > loss_mel_ce: 3.998814344406128 (4.673961322097562) + | > loss: 1.0103672742843628 (1.181892109602843) + | > current_lr: 5e-06 + | > step_time: 0.2099 (0.3073148081710292) + | > loader_time: 0.0086 (0.009188758029838873) + + + --> TIME: 2024-10-26 10:44:11 -- STEP: 19350/30063 -- GLOBAL_STEP: 19350 + | > loss_text_ce: 0.04123488813638687 (0.0535740486001089) + | > loss_mel_ce: 4.072597503662109 (4.672655287281778) + | > loss: 1.0284581184387207 (1.1815573340330925) + | > current_lr: 5e-06 + | > step_time: 0.2621 (0.3073308386913555) + | > loader_time: 0.0084 (0.009188578541580663) + + + --> TIME: 2024-10-26 10:44:37 -- STEP: 19400/30063 -- GLOBAL_STEP: 19400 + | > loss_text_ce: 0.04022153839468956 (0.05354294094132225) + | > loss_mel_ce: 4.594501972198486 (4.671623129721751) + | > loss: 1.1586809158325195 (1.1812915177228511) + | > current_lr: 5e-06 + | > step_time: 0.2049 (0.30735428056765973) + | > loader_time: 0.0098 (0.009188633481251773) + + + --> TIME: 2024-10-26 10:45:02 -- STEP: 19450/30063 -- GLOBAL_STEP: 19450 + | > loss_text_ce: 0.03855489194393158 (0.05351259909206292) + | > loss_mel_ce: 3.98496413230896 (4.670279842783656) + | > loss: 1.0058797597885132 (1.1809481105277335) + | > current_lr: 5e-06 + | > step_time: 0.2673 (0.3073229581960358) + | > loader_time: 0.0091 (0.009188325018993018) + + + --> TIME: 2024-10-26 10:45:28 -- STEP: 19500/30063 -- GLOBAL_STEP: 19500 + | > loss_text_ce: 0.04035898670554161 (0.05347550082244919) + | > loss_mel_ce: 4.22078800201416 (4.668999898653742) + | > loss: 1.0652867555618286 (1.1806188499346748) + | > current_lr: 5e-06 + | > step_time: 0.3737 (0.3073336713130645) + | > loader_time: 0.0094 (0.009187824053642019) + + + --> TIME: 2024-10-26 10:45:54 -- STEP: 19550/30063 -- GLOBAL_STEP: 19550 + | > loss_text_ce: 0.04432866349816322 (0.05344095224900456) + | > loss_mel_ce: 4.526479244232178 (4.667457677885067) + | > loss: 1.1427019834518433 (1.180224657598357) + | > current_lr: 5e-06 + | > step_time: 0.3362 (0.30734027141805087) + | > loader_time: 0.0091 (0.00918779335363436) + + + --> TIME: 2024-10-26 10:46:19 -- STEP: 19600/30063 -- GLOBAL_STEP: 19600 + | > loss_text_ce: 0.04100652039051056 (0.05340917773715832) + | > loss_mel_ce: 4.7256927490234375 (4.666116580440052) + | > loss: 1.191674828529358 (1.17988143960736) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.3073100183083074) + | > loader_time: 0.0087 (0.009187831562392506) + + + --> TIME: 2024-10-26 10:46:44 -- STEP: 19650/30063 -- GLOBAL_STEP: 19650 + | > loss_text_ce: 0.04094251990318298 (0.05337666634726145) + | > loss_mel_ce: 4.526086330413818 (4.664628548294511) + | > loss: 1.1417572498321533 (1.1795013037225344) + | > current_lr: 5e-06 + | > step_time: 0.3789 (0.30727542542319125) + | > loader_time: 0.0092 (0.009188044768859965) + + + --> TIME: 2024-10-26 10:47:08 -- STEP: 19700/30063 -- GLOBAL_STEP: 19700 + | > loss_text_ce: 0.03891918435692787 (0.05334232825237467) + | > loss_mel_ce: 4.481374740600586 (4.6631915526825525) + | > loss: 1.1300734281539917 (1.1791334702883862) + | > current_lr: 5e-06 + | > step_time: 0.2082 (0.3072207989305395) + | > loader_time: 0.009 (0.009188329640983867) + + + --> TIME: 2024-10-26 10:47:33 -- STEP: 19750/30063 -- GLOBAL_STEP: 19750 + | > loss_text_ce: 0.04698820039629936 (0.05331096855541558) + | > loss_mel_ce: 4.23529577255249 (4.66202698322487) + | > loss: 1.070570945739746 (1.1788344879904855) + | > current_lr: 5e-06 + | > step_time: 0.2584 (0.3071934211465363) + | > loader_time: 0.0086 (0.009188182239290965) + + + --> TIME: 2024-10-26 10:47:59 -- STEP: 19800/30063 -- GLOBAL_STEP: 19800 + | > loss_text_ce: 0.04405956715345383 (0.0532794013753005) + | > loss_mel_ce: 4.631063938140869 (4.660861477611016) + | > loss: 1.1687809228897095 (1.1785352197951708) + | > current_lr: 5e-06 + | > step_time: 0.1853 (0.3071987923347575) + | > loader_time: 0.0103 (0.00918792109296773) + + + --> TIME: 2024-10-26 10:48:25 -- STEP: 19850/30063 -- GLOBAL_STEP: 19850 + | > loss_text_ce: 0.03851671516895294 (0.053249370391745564) + | > loss_mel_ce: 3.5309698581695557 (4.659528870330325) + | > loss: 0.892371654510498 (1.1781945602209163) + | > current_lr: 5e-06 + | > step_time: 0.3047 (0.3071965130750704) + | > loader_time: 0.0098 (0.009187988790516827) + + + --> TIME: 2024-10-26 10:48:50 -- STEP: 19900/30063 -- GLOBAL_STEP: 19900 + | > loss_text_ce: 0.04171811789274216 (0.05321625638603401) + | > loss_mel_ce: 4.018099308013916 (4.658145580687092) + | > loss: 1.0149543285369873 (1.177840459301839) + | > current_lr: 5e-06 + | > step_time: 0.2089 (0.3071768002773633) + | > loader_time: 0.0087 (0.009188004738122315) + + + --> TIME: 2024-10-26 10:49:16 -- STEP: 19950/30063 -- GLOBAL_STEP: 19950 + | > loss_text_ce: 0.04186498373746872 (0.05318623755734054) + | > loss_mel_ce: 4.34715461730957 (4.65699904649775) + | > loss: 1.0972548723220825 (1.1775463210401027) + | > current_lr: 5e-06 + | > step_time: 0.262 (0.30717756547425756) + | > loader_time: 0.0097 (0.009187532463169272) + + + --> TIME: 2024-10-26 10:49:43 -- STEP: 20000/30063 -- GLOBAL_STEP: 20000 + | > loss_text_ce: 0.03966930881142616 (0.05315565361399189) + | > loss_mel_ce: 4.5417327880859375 (4.655702110862709) + | > loss: 1.1453505754470825 (1.1772144411295595) + | > current_lr: 5e-06 + | > step_time: 0.3017 (0.30719177695512606) + | > loader_time: 0.0146 (0.00918776695728295) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_20000.pth + + --> TIME: 2024-10-26 10:50:18 -- STEP: 20050/30063 -- GLOBAL_STEP: 20050 + | > loss_text_ce: 0.040113337337970734 (0.05312249194338271) + | > loss_mel_ce: 4.635109901428223 (4.654253286697002) + | > loss: 1.1688058376312256 (1.176843944675007) + | > current_lr: 5e-06 + | > step_time: 0.2776 (0.30716500724640106) + | > loader_time: 0.0086 (0.009189970368458962) + + + --> TIME: 2024-10-26 10:50:43 -- STEP: 20100/30063 -- GLOBAL_STEP: 20100 + | > loss_text_ce: 0.04406638443470001 (0.05309109311884449) + | > loss_mel_ce: 4.593681812286377 (4.652959534599984) + | > loss: 1.1594370603561401 (1.1765126569443027) + | > current_lr: 5e-06 + | > step_time: 0.2604 (0.3071218568531414) + | > loader_time: 0.0091 (0.009190842286864252) + + + --> TIME: 2024-10-26 10:51:09 -- STEP: 20150/30063 -- GLOBAL_STEP: 20150 + | > loss_text_ce: 0.040512971580028534 (0.05305906565365351) + | > loss_mel_ce: 4.103840351104736 (4.65190357727683) + | > loss: 1.0360883474349976 (1.1762406607301252) + | > current_lr: 5e-06 + | > step_time: 0.2997 (0.307124453466522) + | > loader_time: 0.0092 (0.009190465522463139) + + + --> TIME: 2024-10-26 10:51:35 -- STEP: 20200/30063 -- GLOBAL_STEP: 20200 + | > loss_text_ce: 0.03967973217368126 (0.05302752420177097) + | > loss_mel_ce: 3.8816778659820557 (4.650689372827489) + | > loss: 0.9803394079208374 (1.1759292242462043) + | > current_lr: 5e-06 + | > step_time: 0.1878 (0.3071279587485978) + | > loader_time: 0.0086 (0.009190058779008244) + + + --> TIME: 2024-10-26 10:51:59 -- STEP: 20250/30063 -- GLOBAL_STEP: 20250 + | > loss_text_ce: 0.03907120227813721 (0.05299581385689003) + | > loss_mel_ce: 3.5745115280151367 (4.649448014023842) + | > loss: 0.9033956527709961 (1.1756109569514188) + | > current_lr: 5e-06 + | > step_time: 0.3371 (0.30707858004393157) + | > loader_time: 0.0099 (0.009189494780552166) + + + --> TIME: 2024-10-26 10:52:26 -- STEP: 20300/30063 -- GLOBAL_STEP: 20300 + | > loss_text_ce: 0.036256466060876846 (0.05296649813762131) + | > loss_mel_ce: 3.6050047874450684 (4.648316768965677) + | > loss: 0.9103153347969055 (1.1753208167623361) + | > current_lr: 5e-06 + | > step_time: 0.2463 (0.3071224105534272) + | > loader_time: 0.0089 (0.009189334773077677) + + + --> TIME: 2024-10-26 10:52:50 -- STEP: 20350/30063 -- GLOBAL_STEP: 20350 + | > loss_text_ce: 0.040249381214380264 (0.05293420022367245) + | > loss_mel_ce: 3.2418136596679688 (4.6468234605578065) + | > loss: 0.8205157518386841 (1.174939415185393) + | > current_lr: 5e-06 + | > step_time: 0.285 (0.3070755416813856) + | > loader_time: 0.009 (0.009189981312950863) + + + --> TIME: 2024-10-26 10:53:16 -- STEP: 20400/30063 -- GLOBAL_STEP: 20400 + | > loss_text_ce: 0.03565439209342003 (0.05290121401185338) + | > loss_mel_ce: 3.0450804233551025 (4.64562562886404) + | > loss: 0.7701836824417114 (1.17463171071281) + | > current_lr: 5e-06 + | > step_time: 0.2456 (0.3070786636717152) + | > loader_time: 0.0091 (0.00919034364176724) + + + --> TIME: 2024-10-26 10:53:42 -- STEP: 20450/30063 -- GLOBAL_STEP: 20450 + | > loss_text_ce: 0.035617001354694366 (0.05286889268936971) + | > loss_mel_ce: 3.33392333984375 (4.644274364392132) + | > loss: 0.8423851132392883 (1.1742858142753612) + | > current_lr: 5e-06 + | > step_time: 0.257 (0.3070792848731576) + | > loader_time: 0.01 (0.00919097957517807) + + + --> TIME: 2024-10-26 10:54:09 -- STEP: 20500/30063 -- GLOBAL_STEP: 20500 + | > loss_text_ce: 0.03900231793522835 (0.052836796043395126) + | > loss_mel_ce: 3.4845218658447266 (4.642847164642495) + | > loss: 0.8808810710906982 (1.1739209901995706) + | > current_lr: 5e-06 + | > step_time: 0.2713 (0.30709759148155497) + | > loader_time: 0.0091 (0.009191057449433783) + + + --> TIME: 2024-10-26 10:54:34 -- STEP: 20550/30063 -- GLOBAL_STEP: 20550 + | > loss_text_ce: 0.0343870185315609 (0.052806691399836486) + | > loss_mel_ce: 4.404526710510254 (4.641459193392076) + | > loss: 1.109728455543518 (1.1735664712222478) + | > current_lr: 5e-06 + | > step_time: 0.3833 (0.30706847208259486) + | > loader_time: 0.0093 (0.009191071157617792) + + + --> TIME: 2024-10-26 10:55:01 -- STEP: 20600/30063 -- GLOBAL_STEP: 20600 + | > loss_text_ce: 0.038114283233881 (0.052776485104927454) + | > loss_mel_ce: 3.670222759246826 (4.639957707599505) + | > loss: 0.9270842671394348 (1.1731835482074155) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.3071160960081686) + | > loader_time: 0.0085 (0.009190995855238752) + + + --> TIME: 2024-10-26 10:55:26 -- STEP: 20650/30063 -- GLOBAL_STEP: 20650 + | > loss_text_ce: 0.03908388689160347 (0.05274393912025918) + | > loss_mel_ce: 3.702895402908325 (4.638507560829248) + | > loss: 0.9354948401451111 (1.1728128750070175) + | > current_lr: 5e-06 + | > step_time: 0.297 (0.3070936328620053) + | > loader_time: 0.0091 (0.009191087106238185) + + + --> TIME: 2024-10-26 10:55:53 -- STEP: 20700/30063 -- GLOBAL_STEP: 20700 + | > loss_text_ce: 0.03758860379457474 (0.05271460248787248) + | > loss_mel_ce: 3.820145845413208 (4.6373168854782305) + | > loss: 0.9644336104393005 (1.1725078720084632) + | > current_lr: 5e-06 + | > step_time: 0.2233 (0.307122787507833) + | > loader_time: 0.009 (0.009190786771728146) + + + --> TIME: 2024-10-26 10:56:19 -- STEP: 20750/30063 -- GLOBAL_STEP: 20750 + | > loss_text_ce: 0.04117259755730629 (0.052685326438082845) + | > loss_mel_ce: 4.608562469482422 (4.6361938878599185) + | > loss: 1.1624337434768677 (1.1722198035803113) + | > current_lr: 5e-06 + | > step_time: 0.3354 (0.30714138657213425) + | > loader_time: 0.0092 (0.009190648458090101) + + + --> TIME: 2024-10-26 10:56:47 -- STEP: 20800/30063 -- GLOBAL_STEP: 20800 + | > loss_text_ce: 0.03850547596812248 (0.0526558929675508) + | > loss_mel_ce: 4.173282623291016 (4.635117725936241) + | > loss: 1.0529470443725586 (1.1719434047289687) + | > current_lr: 5e-06 + | > step_time: 0.2452 (0.30721105529711545) + | > loader_time: 0.0091 (0.009190512654872973) + + + --> TIME: 2024-10-26 10:57:13 -- STEP: 20850/30063 -- GLOBAL_STEP: 20850 + | > loss_text_ce: 0.04056980088353157 (0.052624620328650486) + | > loss_mel_ce: 4.125818729400635 (4.633705015148165) + | > loss: 1.0415971279144287 (1.171582408866143) + | > current_lr: 5e-06 + | > step_time: 0.3027 (0.3072311692843881) + | > loader_time: 0.009 (0.009190681901195288) + + + --> TIME: 2024-10-26 10:57:38 -- STEP: 20900/30063 -- GLOBAL_STEP: 20900 + | > loss_text_ce: 0.0405069924890995 (0.05259558901787859) + | > loss_mel_ce: 3.822497844696045 (4.632570616411795) + | > loss: 0.9657512307167053 (1.1712915513532505) + | > current_lr: 5e-06 + | > step_time: 0.1748 (0.3072004677576288) + | > loader_time: 0.01 (0.009190755757418482) + + + --> TIME: 2024-10-26 10:58:04 -- STEP: 20950/30063 -- GLOBAL_STEP: 20950 + | > loss_text_ce: 0.04597364366054535 (0.052563360507971064) + | > loss_mel_ce: 4.122974395751953 (4.631258367479246) + | > loss: 1.0422370433807373 (1.1709554320047393) + | > current_lr: 5e-06 + | > step_time: 0.2611 (0.30720688683320674) + | > loader_time: 0.0084 (0.00919065575611044) + + + --> TIME: 2024-10-26 10:58:31 -- STEP: 21000/30063 -- GLOBAL_STEP: 21000 + | > loss_text_ce: 0.03943745419383049 (0.05253277189116024) + | > loss_mel_ce: 3.9351677894592285 (4.629883818671788) + | > loss: 0.9936513304710388 (1.1706041476442632) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.30722976586932005) + | > loader_time: 0.0088 (0.009190231800079274) + + + --> TIME: 2024-10-26 10:58:58 -- STEP: 21050/30063 -- GLOBAL_STEP: 21050 + | > loss_text_ce: 0.03727870434522629 (0.052501528492360956) + | > loss_mel_ce: 4.0858025550842285 (4.6286161357036715) + | > loss: 1.0307703018188477 (1.1702794160525694) + | > current_lr: 5e-06 + | > step_time: 0.2515 (0.3072473909849205) + | > loader_time: 0.008 (0.00918997100866315) + + + --> TIME: 2024-10-26 10:59:23 -- STEP: 21100/30063 -- GLOBAL_STEP: 21100 + | > loss_text_ce: 0.03987004607915878 (0.05247098067281003) + | > loss_mel_ce: 3.704885244369507 (4.627184383349368) + | > loss: 0.936188817024231 (1.169913840994444) + | > current_lr: 5e-06 + | > step_time: 0.3947 (0.3072375922406435) + | > loader_time: 0.0085 (0.009189486503601005) + + + --> TIME: 2024-10-26 10:59:47 -- STEP: 21150/30063 -- GLOBAL_STEP: 21150 + | > loss_text_ce: 0.04270370304584503 (0.05244056863644219) + | > loss_mel_ce: 3.9557247161865234 (4.625894223071132) + | > loss: 0.9996070861816406 (1.1695836979192975) + | > current_lr: 5e-06 + | > step_time: 0.3983 (0.307186076883445) + | > loader_time: 0.0095 (0.009188881867320763) + + + --> TIME: 2024-10-26 11:00:14 -- STEP: 21200/30063 -- GLOBAL_STEP: 21200 + | > loss_text_ce: 0.03610064461827278 (0.05241090966221634) + | > loss_mel_ce: 4.2351274490356445 (4.624847077828509) + | > loss: 1.0678070783615112 (1.1693144968911087) + | > current_lr: 5e-06 + | > step_time: 0.241 (0.30719188773406936) + | > loader_time: 0.0085 (0.009188843403222362) + + + --> TIME: 2024-10-26 11:00:39 -- STEP: 21250/30063 -- GLOBAL_STEP: 21250 + | > loss_text_ce: 0.041630182415246964 (0.052382596805428616) + | > loss_mel_ce: 4.126216888427734 (4.623752812340658) + | > loss: 1.0419617891311646 (1.1690338523079296) + | > current_lr: 5e-06 + | > step_time: 0.2874 (0.30717710018157773) + | > loader_time: 0.0093 (0.009188362671347159) + + + --> TIME: 2024-10-26 11:01:05 -- STEP: 21300/30063 -- GLOBAL_STEP: 21300 + | > loss_text_ce: 0.03775344416499138 (0.05235207514679964) + | > loss_mel_ce: 4.480717182159424 (4.622388183466126) + | > loss: 1.129617691040039 (1.1686850646599845) + | > current_lr: 5e-06 + | > step_time: 0.2037 (0.3071649416511589) + | > loader_time: 0.0082 (0.009187705147434224) + + + --> TIME: 2024-10-26 11:01:30 -- STEP: 21350/30063 -- GLOBAL_STEP: 21350 + | > loss_text_ce: 0.038119133561849594 (0.05232468417180422) + | > loss_mel_ce: 4.122433662414551 (4.621168972814932) + | > loss: 1.0401382446289062 (1.1683734142501048) + | > current_lr: 5e-06 + | > step_time: 0.3129 (0.3071625292329073) + | > loader_time: 0.0092 (0.009187306174070483) + + + --> TIME: 2024-10-26 11:01:56 -- STEP: 21400/30063 -- GLOBAL_STEP: 21400 + | > loss_text_ce: 0.04329284280538559 (0.05229686196345091) + | > loss_mel_ce: 4.662398815155029 (4.62001980051814) + | > loss: 1.1764229536056519 (1.1680791656324725) + | > current_lr: 5e-06 + | > step_time: 0.345 (0.30714895329742575) + | > loader_time: 0.0091 (0.009186642938685133) + + + --> TIME: 2024-10-26 11:02:22 -- STEP: 21450/30063 -- GLOBAL_STEP: 21450 + | > loss_text_ce: 0.03453602269291878 (0.052266238276479686) + | > loss_mel_ce: 3.2324135303497314 (4.618913396821967) + | > loss: 0.8167374134063721 (1.1677949087908692) + | > current_lr: 5e-06 + | > step_time: 0.3785 (0.3071763623066415) + | > loader_time: 0.0087 (0.00918596903483067) + + + --> TIME: 2024-10-26 11:02:49 -- STEP: 21500/30063 -- GLOBAL_STEP: 21500 + | > loss_text_ce: 0.03107278048992157 (0.05223769721007611) + | > loss_mel_ce: 3.9824187755584717 (4.617637216867379) + | > loss: 1.0033729076385498 (1.1674687285450924) + | > current_lr: 5e-06 + | > step_time: 0.2466 (0.3071912837361157) + | > loader_time: 0.0097 (0.009185925672220574) + + + --> TIME: 2024-10-26 11:03:16 -- STEP: 21550/30063 -- GLOBAL_STEP: 21550 + | > loss_text_ce: 0.04106273129582405 (0.05220937618488765) + | > loss_mel_ce: 4.660557746887207 (4.616503187708382) + | > loss: 1.1754051446914673 (1.1671781409920707) + | > current_lr: 5e-06 + | > step_time: 0.3069 (0.30725190902807087) + | > loader_time: 0.0092 (0.00918572351716782) + + + --> TIME: 2024-10-26 11:03:43 -- STEP: 21600/30063 -- GLOBAL_STEP: 21600 + | > loss_text_ce: 0.039408452808856964 (0.05218177615285458) + | > loss_mel_ce: 4.199373245239258 (4.615542037001345) + | > loss: 1.0596954822540283 (1.1669309532918244) + | > current_lr: 5e-06 + | > step_time: 0.2334 (0.30726512268737277) + | > loader_time: 0.0087 (0.009185421577206303) + + + --> TIME: 2024-10-26 11:04:07 -- STEP: 21650/30063 -- GLOBAL_STEP: 21650 + | > loss_text_ce: 0.04326842725276947 (0.05215292413845886) + | > loss_mel_ce: 4.421598434448242 (4.614365937032526) + | > loss: 1.1162166595458984 (1.1666297153127057) + | > current_lr: 5e-06 + | > step_time: 0.2949 (0.30723910588314884) + | > loader_time: 0.0101 (0.009184781012854389) + + + --> TIME: 2024-10-26 11:04:33 -- STEP: 21700/30063 -- GLOBAL_STEP: 21700 + | > loss_text_ce: 0.04013550281524658 (0.052125823927983025) + | > loss_mel_ce: 4.48711633682251 (4.613531345727788) + | > loss: 1.1318129301071167 (1.1664142924409804) + | > current_lr: 5e-06 + | > step_time: 0.2728 (0.3072284306249286) + | > loader_time: 0.0087 (0.009184870049700694) + + + --> TIME: 2024-10-26 11:04:59 -- STEP: 21750/30063 -- GLOBAL_STEP: 21750 + | > loss_text_ce: 0.04418960586190224 (0.05209927699072594) + | > loss_mel_ce: 4.992228984832764 (4.612618956445258) + | > loss: 1.259104609489441 (1.166179558381262) + | > current_lr: 5e-06 + | > step_time: 0.5141 (0.3072325243346968) + | > loader_time: 0.0092 (0.009184153556823666) + + + --> TIME: 2024-10-26 11:05:26 -- STEP: 21800/30063 -- GLOBAL_STEP: 21800 + | > loss_text_ce: 0.038546960800886154 (0.05207121206943035) + | > loss_mel_ce: 4.116634368896484 (4.611553029521867) + | > loss: 1.0387953519821167 (1.1659060604282423) + | > current_lr: 5e-06 + | > step_time: 0.2096 (0.3072610566156696) + | > loader_time: 0.0096 (0.009184523077186021) + + + --> TIME: 2024-10-26 11:05:51 -- STEP: 21850/30063 -- GLOBAL_STEP: 21850 + | > loss_text_ce: 0.03858276829123497 (0.05204223401128414) + | > loss_mel_ce: 4.0600666999816895 (4.610185740937881) + | > loss: 1.0246623754501343 (1.1655569937654657) + | > current_lr: 5e-06 + | > step_time: 0.3738 (0.3072579546651226) + | > loader_time: 0.0086 (0.009184276441141922) + + + --> TIME: 2024-10-26 11:06:17 -- STEP: 21900/30063 -- GLOBAL_STEP: 21900 + | > loss_text_ce: 0.037075821310281754 (0.0520133289704991) + | > loss_mel_ce: 4.262927055358887 (4.608931877819893) + | > loss: 1.0750007629394531 (1.1652363017106113) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.30725410943706016) + | > loader_time: 0.0088 (0.009183614253997734) + + + --> TIME: 2024-10-26 11:06:44 -- STEP: 21950/30063 -- GLOBAL_STEP: 21950 + | > loss_text_ce: 0.035398125648498535 (0.0519866423378432) + | > loss_mel_ce: 2.906306028366089 (4.60793621729064) + | > loss: 0.7354260683059692 (1.1649807149266813) + | > current_lr: 5e-06 + | > step_time: 0.3849 (0.3073070660593271) + | > loader_time: 0.0099 (0.00918322452380058) + + + --> TIME: 2024-10-26 11:07:11 -- STEP: 22000/30063 -- GLOBAL_STEP: 22000 + | > loss_text_ce: 0.03972029685974121 (0.0519575720585204) + | > loss_mel_ce: 4.178533554077148 (4.6066714007420675) + | > loss: 1.0545635223388672 (1.1646572432247009) + | > current_lr: 5e-06 + | > step_time: 0.243 (0.30732843394712683) + | > loader_time: 0.0083 (0.009182742649858584) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804/checkpoint_22000.pth + + --> TIME: 2024-10-26 11:07:48 -- STEP: 22050/30063 -- GLOBAL_STEP: 22050 + | > loss_text_ce: 0.04076101630926132 (0.0519298816769638) + | > loss_mel_ce: 4.162412643432617 (4.6057620146917575) + | > loss: 1.0507934093475342 (1.1644229741053185) + | > current_lr: 5e-06 + | > step_time: 0.2785 (0.30733182931973263) + | > loader_time: 0.0095 (0.0091831833975655) + + + --> TIME: 2024-10-26 11:08:12 -- STEP: 22100/30063 -- GLOBAL_STEP: 22100 + | > loss_text_ce: 0.039491601288318634 (0.0519006260705276) + | > loss_mel_ce: 4.1514716148376465 (4.6041506548272695) + | > loss: 1.0477408170700073 (1.1640128202405933) + | > current_lr: 5e-06 + | > step_time: 0.2075 (0.30729370191625593) + | > loader_time: 0.0084 (0.009182746378005298) + + + --> TIME: 2024-10-26 11:08:38 -- STEP: 22150/30063 -- GLOBAL_STEP: 22150 + | > loss_text_ce: 0.04339059069752693 (0.051872100664416096) + | > loss_mel_ce: 4.635831832885742 (4.602961483410697) + | > loss: 1.169805645942688 (1.1637083960440515) + | > current_lr: 5e-06 + | > step_time: 0.2337 (0.3073072986839556) + | > loader_time: 0.0086 (0.009182630661647897) + + + --> TIME: 2024-10-26 11:09:05 -- STEP: 22200/30063 -- GLOBAL_STEP: 22200 + | > loss_text_ce: 0.0389290377497673 (0.05184510322019176) + | > loss_mel_ce: 4.303701877593994 (4.601942484593568) + | > loss: 1.0856577157974243 (1.1634468969810057) + | > current_lr: 5e-06 + | > step_time: 0.2436 (0.30732055146414655) + | > loader_time: 0.0089 (0.00918244513305451) + + + --> TIME: 2024-10-26 11:09:31 -- STEP: 22250/30063 -- GLOBAL_STEP: 22250 + | > loss_text_ce: 0.035264261066913605 (0.05181751855822748) + | > loss_mel_ce: 3.979304790496826 (4.600691662981231) + | > loss: 1.0036423206329346 (1.1631272954190655) + | > current_lr: 5e-06 + | > step_time: 0.2619 (0.3073471188063002) + | > loader_time: 0.0094 (0.009182179043801914) + + + --> TIME: 2024-10-26 11:09:58 -- STEP: 22300/30063 -- GLOBAL_STEP: 22300 + | > loss_text_ce: 0.03839513286948204 (0.05178987414429928) + | > loss_mel_ce: 3.9766597747802734 (4.599667516738274) + | > loss: 1.0037636756896973 (1.1628643477470826) + | > current_lr: 5e-06 + | > step_time: 0.2758 (0.30737299542790264) + | > loader_time: 0.0093 (0.009182253525396016) + + + --> TIME: 2024-10-26 11:10:23 -- STEP: 22350/30063 -- GLOBAL_STEP: 22350 + | > loss_text_ce: 0.039192430675029755 (0.051761807750001984) + | > loss_mel_ce: 3.331636667251587 (4.598222293181673) + | > loss: 0.8427072763442993 (1.1624960252555903) + | > current_lr: 5e-06 + | > step_time: 0.2668 (0.3073613673941904) + | > loader_time: 0.0088 (0.009181876705263464) + + + --> TIME: 2024-10-26 11:10:49 -- STEP: 22400/30063 -- GLOBAL_STEP: 22400 + | > loss_text_ce: 0.03900080546736717 (0.05173627272747178) + | > loss_mel_ce: 4.584355354309082 (4.597258319620547) + | > loss: 1.1558390855789185 (1.1622486481203507) + | > current_lr: 5e-06 + | > step_time: 0.2332 (0.30735242268868557) + | > loader_time: 0.0086 (0.009181692813123891) + + + --> TIME: 2024-10-26 11:11:17 -- STEP: 22450/30063 -- GLOBAL_STEP: 22450 + | > loss_text_ce: 0.038555581122636795 (0.051710038814685264) + | > loss_mel_ce: 3.456609010696411 (4.596006860074563) + | > loss: 0.8737911581993103 (1.161929224748117) + | > current_lr: 5e-06 + | > step_time: 0.3444 (0.30740114551875264) + | > loader_time: 0.0087 (0.009181055141185512) + + + --> TIME: 2024-10-26 11:11:43 -- STEP: 22500/30063 -- GLOBAL_STEP: 22500 + | > loss_text_ce: 0.039642881602048874 (0.05168120483424915) + | > loss_mel_ce: 3.859917163848877 (4.59486852230493) + | > loss: 0.9748899936676025 (1.161637431814929) + | > current_lr: 5e-06 + | > step_time: 0.2885 (0.3074338149282642) + | > loader_time: 0.0088 (0.009180644787682345) + + + --> TIME: 2024-10-26 11:12:10 -- STEP: 22550/30063 -- GLOBAL_STEP: 22550 + | > loss_text_ce: 0.0403783917427063 (0.051656063996139624) + | > loss_mel_ce: 4.2151947021484375 (4.59392549969402) + | > loss: 1.0638933181762695 (1.161395390953557) + | > current_lr: 5e-06 + | > step_time: 0.2314 (0.3074529030793496) + | > loader_time: 0.0089 (0.009180366025531295) + + + --> TIME: 2024-10-26 11:12:36 -- STEP: 22600/30063 -- GLOBAL_STEP: 22600 + | > loss_text_ce: 0.03614070266485214 (0.05162834915818749) + | > loss_mel_ce: 3.447444200515747 (4.592749804490412) + | > loss: 0.8708962202072144 (1.1610945384323004) + | > current_lr: 5e-06 + | > step_time: 0.2497 (0.3074528685818704) + | > loader_time: 0.0088 (0.009180006812104005) + + + --> TIME: 2024-10-26 11:13:02 -- STEP: 22650/30063 -- GLOBAL_STEP: 22650 + | > loss_text_ce: 0.04471220448613167 (0.05160094608994759) + | > loss_mel_ce: 3.731722593307495 (4.591249055062414) + | > loss: 0.9441087245941162 (1.1607125003079906) + | > current_lr: 5e-06 + | > step_time: 0.4452 (0.3074530590869968) + | > loader_time: 0.0114 (0.00917980549877571) + + + --> TIME: 2024-10-26 11:13:27 -- STEP: 22700/30063 -- GLOBAL_STEP: 22700 + | > loss_text_ce: 0.03793908655643463 (0.05157473423046412) + | > loss_mel_ce: 4.074444770812988 (4.59026586944309) + | > loss: 1.0280959606170654 (1.1604601509245414) + | > current_lr: 5e-06 + | > step_time: 0.229 (0.30744104353866614) + | > loader_time: 0.0087 (0.009179243453273603) + + + --> TIME: 2024-10-26 11:13:53 -- STEP: 22750/30063 -- GLOBAL_STEP: 22750 + | > loss_text_ce: 0.03892311826348305 (0.05154821300801327) + | > loss_mel_ce: 3.962580442428589 (4.589151501666035) + | > loss: 1.0003758668899536 (1.1601749286808751) + | > current_lr: 5e-06 + | > step_time: 0.2995 (0.3074529195198616) + | > loader_time: 0.0086 (0.009178772328974037) + + + --> TIME: 2024-10-26 11:14:19 -- STEP: 22800/30063 -- GLOBAL_STEP: 22800 + | > loss_text_ce: 0.037390001118183136 (0.05152115329577199) + | > loss_mel_ce: 3.642117738723755 (4.587928225973174) + | > loss: 0.9198769330978394 (1.1598623448218108) + | > current_lr: 5e-06 + | > step_time: 0.2908 (0.3074492886505598) + | > loader_time: 0.0087 (0.009178131787400403) + + + --> TIME: 2024-10-26 11:14:46 -- STEP: 22850/30063 -- GLOBAL_STEP: 22850 + | > loss_text_ce: 0.039994265884160995 (0.051493873844163554) + | > loss_mel_ce: 3.982442617416382 (4.586804814578645) + | > loss: 1.0056092739105225 (1.15957467211153) + | > current_lr: 5e-06 + | > step_time: 0.2533 (0.30746673986739276) + | > loader_time: 0.0088 (0.009177784564980172) + + + --> TIME: 2024-10-26 11:15:11 -- STEP: 22900/30063 -- GLOBAL_STEP: 22900 + | > loss_text_ce: 0.04023214429616928 (0.05146728353696867) + | > loss_mel_ce: 4.516982078552246 (4.585872951126497) + | > loss: 1.1393035650253296 (1.1593350586672466) + | > current_lr: 5e-06 + | > step_time: 0.3547 (0.3074554911034565) + | > loader_time: 0.0098 (0.009177307083096468) + + + --> TIME: 2024-10-26 11:15:37 -- STEP: 22950/30063 -- GLOBAL_STEP: 22950 + | > loss_text_ce: 0.0337962880730629 (0.05144212600923169) + | > loss_mel_ce: 3.939302444458008 (4.584859523949786) + | > loss: 0.9932746887207031 (1.1590754124983127) + | > current_lr: 5e-06 + | > step_time: 0.3011 (0.30746196927587666) + | > loader_time: 0.0087 (0.009177019393521868) + + + --> TIME: 2024-10-26 11:16:05 -- STEP: 23000/30063 -- GLOBAL_STEP: 23000 + | > loss_text_ce: 0.03473746404051781 (0.051417191869538637) + | > loss_mel_ce: 3.8611998558044434 (4.583839598987401) + | > loss: 0.9739843010902405 (1.1588141976905826) + | > current_lr: 5e-06 + | > step_time: 0.2039 (0.30752189105489697) + | > loader_time: 0.0087 (0.009177071167075005) + + + --> TIME: 2024-10-26 11:16:31 -- STEP: 23050/30063 -- GLOBAL_STEP: 23050 + | > loss_text_ce: 0.03577152639627457 (0.05139038101767951) + | > loss_mel_ce: 4.094619274139404 (4.582444404500669) + | > loss: 1.0325976610183716 (1.1584586963710368) + | > current_lr: 5e-06 + | > step_time: 0.2498 (0.30753614650113703) + | > loader_time: 0.0086 (0.00917686929930305) + + + --> TIME: 2024-10-26 11:16:58 -- STEP: 23100/30063 -- GLOBAL_STEP: 23100 + | > loss_text_ce: 0.034574348479509354 (0.05136337273185697) + | > loss_mel_ce: 3.6973581314086914 (4.581135695950795) + | > loss: 0.9329831004142761 (1.158124767162576) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.3075535496488766) + | > loader_time: 0.0084 (0.009176862642362404) + + + --> TIME: 2024-10-26 11:17:25 -- STEP: 23150/30063 -- GLOBAL_STEP: 23150 + | > loss_text_ce: 0.040144145488739014 (0.051336818907498284) + | > loss_mel_ce: 5.011302471160889 (4.580022749179883) + | > loss: 1.2628616094589233 (1.1578398920109598) + | > current_lr: 5e-06 + | > step_time: 0.3814 (0.30759605905149445) + | > loader_time: 0.009 (0.009176449023878992) + + + --> TIME: 2024-10-26 11:17:52 -- STEP: 23200/30063 -- GLOBAL_STEP: 23200 + | > loss_text_ce: 0.03834669664502144 (0.05131158359485248) + | > loss_mel_ce: 3.7772135734558105 (4.578856437195981) + | > loss: 0.9538900852203369 (1.1575420051967213) + | > current_lr: 5e-06 + | > step_time: 0.2024 (0.30761116517000925) + | > loader_time: 0.0088 (0.009175946301427262) + + + --> TIME: 2024-10-26 11:18:18 -- STEP: 23250/30063 -- GLOBAL_STEP: 23250 + | > loss_text_ce: 0.03806125745177269 (0.051285548965376425) + | > loss_mel_ce: 3.950366735458374 (4.57773058886168) + | > loss: 0.9971069693565369 (1.1572540344525344) + | > current_lr: 5e-06 + | > step_time: 0.2943 (0.307623027893802) + | > loader_time: 0.0093 (0.009175439260339101) + + + --> TIME: 2024-10-26 11:18:43 -- STEP: 23300/30063 -- GLOBAL_STEP: 23300 + | > loss_text_ce: 0.04195563867688179 (0.05126084982717293) + | > loss_mel_ce: 4.597232818603516 (4.576497005843289) + | > loss: 1.1597970724105835 (1.1569394639095965) + | > current_lr: 5e-06 + | > step_time: 0.2758 (0.3076116613257058) + | > loader_time: 0.0089 (0.009175371274415908) + + + --> TIME: 2024-10-26 11:19:10 -- STEP: 23350/30063 -- GLOBAL_STEP: 23350 + | > loss_text_ce: 0.03810012713074684 (0.051236621901249224) + | > loss_mel_ce: 4.161343574523926 (4.575368474637661) + | > loss: 1.049860954284668 (1.1566512741419621) + | > current_lr: 5e-06 + | > step_time: 0.368 (0.307621268838067) + | > loader_time: 0.0092 (0.009175081396000682) + + + --> TIME: 2024-10-26 11:19:35 -- STEP: 23400/30063 -- GLOBAL_STEP: 23400 + | > loss_text_ce: 0.03793520852923393 (0.05120980165365466) + | > loss_mel_ce: 4.491229057312012 (4.574332157139072) + | > loss: 1.1322910785675049 (1.1563854897047656) + | > current_lr: 5e-06 + | > step_time: 0.2723 (0.3076002011747415) + | > loader_time: 0.0082 (0.009174830556934592) + + + --> TIME: 2024-10-26 11:20:00 -- STEP: 23450/30063 -- GLOBAL_STEP: 23450 + | > loss_text_ce: 0.04030856862664223 (0.05118276008243932) + | > loss_mel_ce: 4.5825114250183105 (4.573169192728952) + | > loss: 1.1557049751281738 (1.1560879882058042) + | > current_lr: 5e-06 + | > step_time: 0.2767 (0.30757550726566985) + | > loader_time: 0.0099 (0.00917470395183755) + + + --> TIME: 2024-10-26 11:20:25 -- STEP: 23500/30063 -- GLOBAL_STEP: 23500 + | > loss_text_ce: 0.04668625816702843 (0.0511576143480202) + | > loss_mel_ce: 4.422684192657471 (4.571909707789714) + | > loss: 1.1173425912857056 (1.1557668305356392) + | > current_lr: 5e-06 + | > step_time: 0.2299 (0.30754784616510644) + | > loader_time: 0.0105 (0.009174406051635624) + + + --> TIME: 2024-10-26 11:20:50 -- STEP: 23550/30063 -- GLOBAL_STEP: 23550 + | > loss_text_ce: 0.03873784467577934 (0.051131246718805305) + | > loss_mel_ce: 3.663515329360962 (4.570699267964443) + | > loss: 0.9255632758140564 (1.1554576286743088) + | > current_lr: 5e-06 + | > step_time: 0.2851 (0.30754387547762) + | > loader_time: 0.0086 (0.009173715322133971) + + + --> TIME: 2024-10-26 11:21:16 -- STEP: 23600/30063 -- GLOBAL_STEP: 23600 + | > loss_text_ce: 0.037237897515296936 (0.05110584732658075) + | > loss_mel_ce: 3.71714448928833 (4.5692363411895185) + | > loss: 0.9385955929756165 (1.1550855471294752) + | > current_lr: 5e-06 + | > step_time: 0.2029 (0.30752015347197637) + | > loader_time: 0.0081 (0.009173140434895415) + + + --> TIME: 2024-10-26 11:21:40 -- STEP: 23650/30063 -- GLOBAL_STEP: 23650 + | > loss_text_ce: 0.04424761235713959 (0.0510818488758064) + | > loss_mel_ce: 4.385425567626953 (4.568124616856799) + | > loss: 1.1074182987213135 (1.1548016164368289) + | > current_lr: 5e-06 + | > step_time: 0.337 (0.30748736388587283) + | > loader_time: 0.0087 (0.009172417457209456) + + + --> TIME: 2024-10-26 11:22:06 -- STEP: 23700/30063 -- GLOBAL_STEP: 23700 + | > loss_text_ce: 0.043674536049366 (0.05105697165255104) + | > loss_mel_ce: 4.3470916748046875 (4.567111886402716) + | > loss: 1.097691535949707 (1.1545422145334416) + | > current_lr: 5e-06 + | > step_time: 0.3495 (0.30748174791094385) + | > loader_time: 0.0098 (0.009171878899199916) + + ! Run is kept in /kaggle/temp/checkpoints/GPT_XTTS_FT-October-26-2024_07+55AM-633d804