diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,3449 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 4 + | > Num. of Torch Threads: 1 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=/kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b + + > Model has 522165080 parameters + + > EPOCH: 0/5 + --> /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b + + > TRAINING (2024-10-20 17:26:40)  + + --> TIME: 2024-10-20 17:26:44 -- STEP: 0/16928 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.034479375928640366 (0.034479375928640366) + | > loss_mel_ce: 5.541518211364746 (5.541518211364746) + | > loss: 1.3939993381500244 (1.3939993381500244) + | > current_lr: 5e-06 + | > step_time: 0.7977 (0.7977089881896973) + | > loader_time: 3.3924 (3.392357349395752) + + + --> TIME: 2024-10-20 17:27:12 -- STEP: 50/16928 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.03731550648808479 (0.036023978143930435) + | > loss_mel_ce: 4.302874565124512 (5.076376132965088) + | > loss: 1.0850474834442139 (1.2781000232696533) + | > current_lr: 5e-06 + | > step_time: 0.4604 (0.335532922744751) + | > loader_time: 0.018 (0.015007314682006835) + + + --> TIME: 2024-10-20 17:27:39 -- STEP: 100/16928 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.03629765287041664 (0.036043168418109416) + | > loss_mel_ce: 5.084025859832764 (4.991961946487425) + | > loss: 1.2800809144973755 (1.2570012724399569) + | > current_lr: 5e-06 + | > step_time: 0.4167 (0.32793174982070916) + | > loader_time: 0.0112 (0.01412785530090332) + + + --> TIME: 2024-10-20 17:28:05 -- STEP: 150/16928 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.03614922985434532 (0.03611867904663086) + | > loss_mel_ce: 4.862426280975342 (4.962502500216166) + | > loss: 1.2246438264846802 (1.2496552912394208) + | > current_lr: 5e-06 + | > step_time: 0.3352 (0.3213447856903075) + | > loader_time: 0.0142 (0.013893504142761231) + + + --> TIME: 2024-10-20 17:28:32 -- STEP: 200/16928 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.03521576523780823 (0.03616892662830651) + | > loss_mel_ce: 4.869055271148682 (4.923769203424453) + | > loss: 1.2260677814483643 (1.2399845290184022) + | > current_lr: 5e-06 + | > step_time: 0.1536 (0.3230862939357758) + | > loader_time: 0.0106 (0.01347239375114441) + + + --> TIME: 2024-10-20 17:28:59 -- STEP: 250/16928 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.03492993116378784 (0.0361942038461566) + | > loss_mel_ce: 4.392003059387207 (4.872181675910948) + | > loss: 1.1067332029342651 (1.227093966960907) + | > current_lr: 5e-06 + | > step_time: 0.3407 (0.3211192655563355) + | > loader_time: 0.0097 (0.013119044303894043) + + + --> TIME: 2024-10-20 17:29:24 -- STEP: 300/16928 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.03497900813817978 (0.03618900345638394) + | > loss_mel_ce: 4.867762088775635 (4.841025998592373) + | > loss: 1.2256852388381958 (1.2193037466208145) + | > current_lr: 5e-06 + | > step_time: 0.2335 (0.31620238463083905) + | > loader_time: 0.0084 (0.012823618253072102) + + + --> TIME: 2024-10-20 17:29:51 -- STEP: 350/16928 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.03308403119444847 (0.036143964709980124) + | > loss_mel_ce: 4.368578910827637 (4.8056094224112345) + | > loss: 1.1004157066345215 (1.210438344138009) + | > current_lr: 5e-06 + | > step_time: 0.3569 (0.31765812260763987) + | > loader_time: 0.0142 (0.012652753421238491) + + + --> TIME: 2024-10-20 17:30:16 -- STEP: 400/16928 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.03565486520528793 (0.03611913083586843) + | > loss_mel_ce: 4.722887992858887 (4.778911167383189) + | > loss: 1.1896357536315918 (1.2037575732171535) + | > current_lr: 5e-06 + | > step_time: 0.1885 (0.3150532191991803) + | > loader_time: 0.0123 (0.012452394366264344) + + + --> TIME: 2024-10-20 17:30:44 -- STEP: 450/16928 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.03303706645965576 (0.03613451285080778) + | > loss_mel_ce: 4.822587966918945 (4.754940964380894) + | > loss: 1.2139062881469727 (1.1977688679430225) + | > current_lr: 5e-06 + | > step_time: 0.4595 (0.31645603497823055) + | > loader_time: 0.0184 (0.012331858211093479) + + + --> TIME: 2024-10-20 17:31:11 -- STEP: 500/16928 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.03906397148966789 (0.03614925818890334) + | > loss_mel_ce: 4.982487201690674 (4.737125676155085) + | > loss: 1.255387783050537 (1.1933187325000767) + | > current_lr: 5e-06 + | > step_time: 0.2539 (0.31706215667724585) + | > loader_time: 0.0107 (0.012223148345947265) + + + --> TIME: 2024-10-20 17:31:39 -- STEP: 550/16928 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.034849852323532104 (0.03610896364531734) + | > loss_mel_ce: 4.208927631378174 (4.721551756858819) + | > loss: 1.0609443187713623 (1.1894151787324392) + | > current_lr: 5e-06 + | > step_time: 0.5088 (0.31834977626800515) + | > loader_time: 0.0113 (0.012119590585882014) + + + --> TIME: 2024-10-20 17:32:06 -- STEP: 600/16928 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.03654540330171585 (0.03611666072780889) + | > loss_mel_ce: 4.014219284057617 (4.7056853584448435) + | > loss: 1.0126911401748657 (1.1854505035281189) + | > current_lr: 5e-06 + | > step_time: 0.3446 (0.31825836857159917) + | > loader_time: 0.0096 (0.011971223354339606) + + + --> TIME: 2024-10-20 17:32:31 -- STEP: 650/16928 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.037688955664634705 (0.036173603179363134) + | > loss_mel_ce: 4.755973815917969 (4.688844737639788) + | > loss: 1.1984156370162964 (1.1812545846058773) + | > current_lr: 5e-06 + | > step_time: 0.508 (0.31625807432027964) + | > loader_time: 0.0089 (0.01190664694859432) + + + --> TIME: 2024-10-20 17:32:55 -- STEP: 700/16928 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.03294135630130768 (0.03612269162599532) + | > loss_mel_ce: 4.186535358428955 (4.675737782887046) + | > loss: 1.0548691749572754 (1.1779651177780974) + | > current_lr: 5e-06 + | > step_time: 0.2048 (0.31418729611805524) + | > loader_time: 0.0095 (0.011797533375876296) + + + --> TIME: 2024-10-20 17:33:22 -- STEP: 750/16928 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.03753974661231041 (0.036081931290527176) + | > loss_mel_ce: 4.800288200378418 (4.666798933664949) + | > loss: 1.2094570398330688 (1.1757202157179518) + | > current_lr: 5e-06 + | > step_time: 0.4423 (0.31486855443318695) + | > loader_time: 0.0147 (0.01172649478912354) + + + --> TIME: 2024-10-20 17:33:49 -- STEP: 800/16928 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.03657739236950874 (0.0360683992342092) + | > loss_mel_ce: 4.658847808837891 (4.658783264756198) + | > loss: 1.173856258392334 (1.1737129157036543) + | > current_lr: 5e-06 + | > step_time: 0.4178 (0.3146750992536545) + | > loader_time: 0.0112 (0.01161537885665894) + + + --> TIME: 2024-10-20 17:34:15 -- STEP: 850/16928 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.036753587424755096 (0.03610849163768924) + | > loss_mel_ce: 4.373823642730713 (4.648784614450784) + | > loss: 1.1026443243026733 (1.1712232765029456) + | > current_lr: 5e-06 + | > step_time: 0.2543 (0.31475612612331616) + | > loader_time: 0.0089 (0.011538963037378652) + + + --> TIME: 2024-10-20 17:34:43 -- STEP: 900/16928 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.03549529239535332 (0.036090834012462054) + | > loss_mel_ce: 4.775046348571777 (4.637814423773018) + | > loss: 1.202635407447815 (1.1684763138161764) + | > current_lr: 5e-06 + | > step_time: 0.2296 (0.31585186031129625) + | > loader_time: 0.0091 (0.011465948952568906) + + + --> TIME: 2024-10-20 17:35:10 -- STEP: 950/16928 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.03230704367160797 (0.03609395293420869) + | > loss_mel_ce: 4.435161113739014 (4.6297791388160245) + | > loss: 1.1168670654296875 (1.1664682722719093) + | > current_lr: 5e-06 + | > step_time: 0.3378 (0.3159537340465345) + | > loader_time: 0.0102 (0.011418499193693466) + + + --> TIME: 2024-10-20 17:35:36 -- STEP: 1000/16928 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.0363365039229393 (0.0360688256248832) + | > loss_mel_ce: 4.40908670425415 (4.623503723859781) + | > loss: 1.1113557815551758 (1.164893136560917) + | > current_lr: 5e-06 + | > step_time: 0.2255 (0.31581021428108214) + | > loader_time: 0.0099 (0.01135081505775452) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_1000.pth + + --> TIME: 2024-10-20 17:36:15 -- STEP: 1050/16928 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.03643175587058067 (0.03608981916592236) + | > loss_mel_ce: 4.236586570739746 (4.614849513825911) + | > loss: 1.0682545900344849 (1.1627348325933728) + | > current_lr: 5e-06 + | > step_time: 0.2543 (0.31652182601747064) + | > loader_time: 0.0103 (0.011315194084530788) + + + --> TIME: 2024-10-20 17:36:40 -- STEP: 1100/16928 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.03883042559027672 (0.03605694604529576) + | > loss_mel_ce: 4.009004592895508 (4.607236309268254) + | > loss: 1.0119587182998657 (1.1608233136480506) + | > current_lr: 5e-06 + | > step_time: 0.2366 (0.3160727401213212) + | > loader_time: 0.0099 (0.01125429586930709) + + + --> TIME: 2024-10-20 17:37:06 -- STEP: 1150/16928 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.03725716099143028 (0.03602565535708613) + | > loss_mel_ce: 4.476228713989258 (4.599000332873798) + | > loss: 1.1283714771270752 (1.1587564967507902) + | > current_lr: 5e-06 + | > step_time: 0.3043 (0.31531839039014736) + | > loader_time: 0.0092 (0.011205499068550447) + + + --> TIME: 2024-10-20 17:37:34 -- STEP: 1200/16928 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.03444804251194 (0.03601582252730926) + | > loss_mel_ce: 4.127522945404053 (4.593374021053314) + | > loss: 1.0404927730560303 (1.1573474606871599) + | > current_lr: 5e-06 + | > step_time: 0.2447 (0.31629985153675105) + | > loader_time: 0.0093 (0.011184990803400682) + + + --> TIME: 2024-10-20 17:38:02 -- STEP: 1250/16928 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.035586606711149216 (0.03601967471539973) + | > loss_mel_ce: 4.519235134124756 (4.587300177764887) + | > loss: 1.1387054920196533 (1.1558299631118762) + | > current_lr: 5e-06 + | > step_time: 0.3007 (0.3171365829467776) + | > loader_time: 0.0089 (0.011142688560485848) + + + --> TIME: 2024-10-20 17:38:29 -- STEP: 1300/16928 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.03692324832081795 (0.036018457739399025) + | > loss_mel_ce: 4.427443504333496 (4.5808082498036855) + | > loss: 1.1160917282104492 (1.15420667694165) + | > current_lr: 5e-06 + | > step_time: 0.3563 (0.3174149019901571) + | > loader_time: 0.012 (0.011094063795529886) + + + --> TIME: 2024-10-20 17:38:57 -- STEP: 1350/16928 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.0327133946120739 (0.03603436048935962) + | > loss_mel_ce: 4.362990379333496 (4.576040903197393) + | > loss: 1.0989259481430054 (1.153018815782333) + | > current_lr: 5e-06 + | > step_time: 0.4637 (0.3177816970260056) + | > loader_time: 0.011 (0.011059710184733078) + + + --> TIME: 2024-10-20 17:39:24 -- STEP: 1400/16928 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.03751272335648537 (0.036022246839212554) + | > loss_mel_ce: 4.632017135620117 (4.571183800356727) + | > loss: 1.1673824787139893 (1.1518015115175912) + | > current_lr: 5e-06 + | > step_time: 0.3206 (0.317783194439752) + | > loader_time: 0.0124 (0.01101673194340297) + + + --> TIME: 2024-10-20 17:39:51 -- STEP: 1450/16928 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.03643741086125374 (0.03602829539698776) + | > loss_mel_ce: 4.310417652130127 (4.5635494171339905) + | > loss: 1.0867137908935547 (1.1498944279654253) + | > current_lr: 5e-06 + | > step_time: 0.2504 (0.3179833446699999) + | > loader_time: 0.0096 (0.010974855751826842) + + + --> TIME: 2024-10-20 17:40:17 -- STEP: 1500/16928 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.042186517268419266 (0.03601582924649118) + | > loss_mel_ce: 4.711681365966797 (4.557342652797702) + | > loss: 1.188467025756836 (1.1483396205504726) + | > current_lr: 5e-06 + | > step_time: 0.1843 (0.317879419008891) + | > loader_time: 0.0086 (0.010938528219858802) + + + --> TIME: 2024-10-20 17:40:43 -- STEP: 1550/16928 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.03163200616836548 (0.03600447414502979) + | > loss_mel_ce: 4.3515825271606445 (4.551207505041559) + | > loss: 1.0958036184310913 (1.1468029949357428) + | > current_lr: 5e-06 + | > step_time: 0.2268 (0.3177368617826896) + | > loader_time: 0.0114 (0.010916813112074326) + + + --> TIME: 2024-10-20 17:41:09 -- STEP: 1600/16928 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.035571105778217316 (0.03600356907700189) + | > loss_mel_ce: 4.290834903717041 (4.543948182463652) + | > loss: 1.0816015005111694 (1.1449879380315535) + | > current_lr: 5e-06 + | > step_time: 0.3049 (0.31740963995456734) + | > loader_time: 0.0137 (0.010895385444164274) + + + --> TIME: 2024-10-20 17:41:36 -- STEP: 1650/16928 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.036737293004989624 (0.03598632165428368) + | > loss_mel_ce: 4.558343887329102 (4.538656435446313) + | > loss: 1.1487703323364258 (1.1436606892672454) + | > current_lr: 5e-06 + | > step_time: 0.2709 (0.31745655970139963) + | > loader_time: 0.0121 (0.010871404156540377) + + + --> TIME: 2024-10-20 17:42:04 -- STEP: 1700/16928 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.03393169119954109 (0.03598436059101546) + | > loss_mel_ce: 4.200849533081055 (4.532909435103933) + | > loss: 1.0586953163146973 (1.14222344903385) + | > current_lr: 5e-06 + | > step_time: 0.2552 (0.31782989445854665) + | > loader_time: 0.0107 (0.010841774800244498) + + + --> TIME: 2024-10-20 17:42:30 -- STEP: 1750/16928 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.030179357156157494 (0.03599445923204941) + | > loss_mel_ce: 4.398261547088623 (4.527194279806965) + | > loss: 1.1071102619171143 (1.1407971847397953) + | > current_lr: 5e-06 + | > step_time: 0.3794 (0.317823970794678) + | > loader_time: 0.0092 (0.010809077262878415) + + + --> TIME: 2024-10-20 17:42:56 -- STEP: 1800/16928 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.036965422332286835 (0.03599940845639351) + | > loss_mel_ce: 4.304959297180176 (4.522654445966097) + | > loss: 1.0854811668395996 (1.1396634634004714) + | > current_lr: 5e-06 + | > step_time: 0.2244 (0.317517364157571) + | > loader_time: 0.01 (0.010774941311942204) + + + --> TIME: 2024-10-20 17:43:23 -- STEP: 1850/16928 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.034721989184617996 (0.03600018272528786) + | > loss_mel_ce: 4.164417743682861 (4.515258112082622) + | > loss: 1.0497848987579346 (1.1378145735650464) + | > current_lr: 5e-06 + | > step_time: 0.3359 (0.31762732634673274) + | > loader_time: 0.009 (0.010749731450467495) + + + --> TIME: 2024-10-20 17:43:51 -- STEP: 1900/16928 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.03528560325503349 (0.03597724632801203) + | > loss_mel_ce: 4.133517265319824 (4.511459486233572) + | > loss: 1.0422006845474243 (1.13685918280953) + | > current_lr: 5e-06 + | > step_time: 0.3229 (0.3178276274078773) + | > loader_time: 0.0108 (0.010727329003183463) + + + --> TIME: 2024-10-20 17:44:19 -- STEP: 1950/16928 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.03669888153672218 (0.03597703789671271) + | > loss_mel_ce: 4.258001804351807 (4.508477535125548) + | > loss: 1.0736751556396484 (1.136113643034912) + | > current_lr: 5e-06 + | > step_time: 0.5221 (0.3185312156188185) + | > loader_time: 0.012 (0.010712036475157125) + + + --> TIME: 2024-10-20 17:44:47 -- STEP: 2000/16928 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.036615271121263504 (0.035969742706045596) + | > loss_mel_ce: 4.786861419677734 (4.5051460707187765) + | > loss: 1.205869197845459 (1.1352789531052125) + | > current_lr: 5e-06 + | > step_time: 0.2466 (0.3188162286281588) + | > loader_time: 0.0092 (0.010690343022346494) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_2000.pth + + --> TIME: 2024-10-20 17:45:27 -- STEP: 2050/16928 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.03466219827532768 (0.03596415305828184) + | > loss_mel_ce: 3.9935576915740967 (4.5007856677218285) + | > loss: 1.0070549249649048 (1.134187454944705) + | > current_lr: 5e-06 + | > step_time: 0.2523 (0.3184648407959359) + | > loader_time: 0.009 (0.010695361160650485) + + + --> TIME: 2024-10-20 17:45:54 -- STEP: 2100/16928 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.03808470070362091 (0.035943749736817195) + | > loss_mel_ce: 4.2260003089904785 (4.498251244227101) + | > loss: 1.066021203994751 (1.1335487483796631) + | > current_lr: 5e-06 + | > step_time: 0.2243 (0.31838244426818146) + | > loader_time: 0.0081 (0.010660786628723143) + + + --> TIME: 2024-10-20 17:46:21 -- STEP: 2150/16928 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.03869171813130379 (0.03594591626420972) + | > loss_mel_ce: 4.534929275512695 (4.494279400137977) + | > loss: 1.1434051990509033 (1.1325563290507306) + | > current_lr: 5e-06 + | > step_time: 0.3087 (0.318664125065471) + | > loader_time: 0.0088 (0.010632263782412506) + + + --> TIME: 2024-10-20 17:46:46 -- STEP: 2200/16928 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.032937534153461456 (0.035946108233183706) + | > loss_mel_ce: 4.061943531036377 (4.489286118203952) + | > loss: 1.0237202644348145 (1.1313080565766862) + | > current_lr: 5e-06 + | > step_time: 0.3207 (0.3180325853824618) + | > loader_time: 0.0089 (0.010614502863450481) + + + --> TIME: 2024-10-20 17:47:13 -- STEP: 2250/16928 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.03791983425617218 (0.03594557960248664) + | > loss_mel_ce: 4.871583938598633 (4.486278942002199) + | > loss: 1.2273759841918945 (1.1305561305152054) + | > current_lr: 5e-06 + | > step_time: 0.4286 (0.318142015881009) + | > loader_time: 0.0097 (0.010594862620035804) + + + --> TIME: 2024-10-20 17:47:40 -- STEP: 2300/16928 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.03303423151373863 (0.035948890969967166) + | > loss_mel_ce: 4.533657073974609 (4.481799644490953) + | > loss: 1.1416728496551514 (1.129437133877175) + | > current_lr: 5e-06 + | > step_time: 0.3225 (0.3182524020775508) + | > loader_time: 0.009 (0.010571613311767572) + + + --> TIME: 2024-10-20 17:48:07 -- STEP: 2350/16928 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.03692504018545151 (0.03594023171336737) + | > loss_mel_ce: 4.6282057762146 (4.47936522047571) + | > loss: 1.1662826538085938 (1.128826362959883) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.3184197983843219) + | > loader_time: 0.0103 (0.010561981302626582) + + + --> TIME: 2024-10-20 17:48:35 -- STEP: 2400/16928 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.03533298894762993 (0.035946600000218897) + | > loss_mel_ce: 4.483459949493408 (4.475395907362312) + | > loss: 1.1296982765197754 (1.127835626776021) + | > current_lr: 5e-06 + | > step_time: 0.2898 (0.318595876693726) + | > loader_time: 0.0108 (0.010542380015055338) + + + --> TIME: 2024-10-20 17:49:02 -- STEP: 2450/16928 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.03867911174893379 (0.035947128164068885) + | > loss_mel_ce: 4.363792419433594 (4.470566223202929) + | > loss: 1.1006178855895996 (1.1266283377579303) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.3187910183108587) + | > loader_time: 0.0091 (0.010524018929929149) + + + --> TIME: 2024-10-20 17:49:29 -- STEP: 2500/16928 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.03406631946563721 (0.03594519359469423) + | > loss_mel_ce: 5.012501239776611 (4.4676113759994625) + | > loss: 1.2616418600082397 (1.1258891422033332) + | > current_lr: 5e-06 + | > step_time: 0.3546 (0.31881914701461833) + | > loader_time: 0.0102 (0.010508809661865232) + + + --> TIME: 2024-10-20 17:49:57 -- STEP: 2550/16928 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.03824709355831146 (0.03593844796804832) + | > loss_mel_ce: 3.5046896934509277 (4.462937834029115) + | > loss: 0.8857342004776001 (1.1247190703130256) + | > current_lr: 5e-06 + | > step_time: 0.2893 (0.31918782608181856) + | > loader_time: 0.0093 (0.01049493593328139) + + + --> TIME: 2024-10-20 17:50:23 -- STEP: 2600/16928 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.03258243575692177 (0.035932652256093475) + | > loss_mel_ce: 4.066554546356201 (4.459234556601611) + | > loss: 1.0247842073440552 (1.1237918020211755) + | > current_lr: 5e-06 + | > step_time: 0.1552 (0.31899357062119754) + | > loader_time: 0.009 (0.010481428458140443) + + + --> TIME: 2024-10-20 17:50:49 -- STEP: 2650/16928 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.03734006732702255 (0.03593078625160018) + | > loss_mel_ce: 4.23488712310791 (4.455695689489276) + | > loss: 1.0680568218231201 (1.1229066188380423) + | > current_lr: 5e-06 + | > step_time: 0.4447 (0.318881641243989) + | > loader_time: 0.0095 (0.010460542552876022) + + + --> TIME: 2024-10-20 17:51:17 -- STEP: 2700/16928 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.03557657077908516 (0.035929291149670445) + | > loss_mel_ce: 4.021712303161621 (4.4527029447202535) + | > loss: 1.0143221616744995 (1.1221580589038374) + | > current_lr: 5e-06 + | > step_time: 0.2736 (0.3191353020844637) + | > loader_time: 0.0102 (0.01044624761298851) + + + --> TIME: 2024-10-20 17:51:43 -- STEP: 2750/16928 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.036651890724897385 (0.035919731916351845) + | > loss_mel_ce: 4.1645684242248535 (4.449376005086049) + | > loss: 1.0503051280975342 (1.1213239341865913) + | > current_lr: 5e-06 + | > step_time: 0.3119 (0.318923366026445) + | > loader_time: 0.009 (0.010427489367398353) + + + --> TIME: 2024-10-20 17:52:10 -- STEP: 2800/16928 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.03399019315838814 (0.0359173954876939) + | > loss_mel_ce: 4.453664779663086 (4.4453029569557945) + | > loss: 1.1219137907028198 (1.1203050880559868) + | > current_lr: 5e-06 + | > step_time: 0.2717 (0.31906562600817023) + | > loader_time: 0.0096 (0.010410110524722507) + + + --> TIME: 2024-10-20 17:52:38 -- STEP: 2850/16928 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.03348027914762497 (0.03591254447113012) + | > loss_mel_ce: 4.567329406738281 (4.442324836463281) + | > loss: 1.1502023935317993 (1.119559345308104) + | > current_lr: 5e-06 + | > step_time: 0.25 (0.31934445548475837) + | > loader_time: 0.0086 (0.010390153014869024) + + + --> TIME: 2024-10-20 17:53:06 -- STEP: 2900/16928 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.03839527815580368 (0.03591824659242718) + | > loss_mel_ce: 4.406687259674072 (4.4397345526465255) + | > loss: 1.1112706661224365 (1.1189131998399215) + | > current_lr: 5e-06 + | > step_time: 0.1758 (0.3195617693868174) + | > loader_time: 0.0097 (0.010375626498255238) + + + --> TIME: 2024-10-20 17:53:33 -- STEP: 2950/16928 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.036969512701034546 (0.03590795128969322) + | > loss_mel_ce: 4.311365127563477 (4.436569091748401) + | > loss: 1.0870836973190308 (1.1181192608202928) + | > current_lr: 5e-06 + | > step_time: 0.4653 (0.31968891814603634) + | > loader_time: 0.01 (0.010358441482156021) + + + --> TIME: 2024-10-20 17:54:01 -- STEP: 3000/16928 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.035886604338884354 (0.03590260061373317) + | > loss_mel_ce: 4.435675144195557 (4.433228038628912) + | > loss: 1.117890477180481 (1.1172826599280057) + | > current_lr: 5e-06 + | > step_time: 0.2715 (0.3198408749898274) + | > loader_time: 0.0089 (0.010341001749038705) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_3000.pth + + --> TIME: 2024-10-20 17:54:40 -- STEP: 3050/16928 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.03722637891769409 (0.03588889550967296) + | > loss_mel_ce: 4.805922508239746 (4.431634678371633) + | > loss: 1.2107871770858765 (1.116880893550938) + | > current_lr: 5e-06 + | > step_time: 0.3639 (0.3195883750133824) + | > loader_time: 0.0101 (0.010335988060372784) + + + --> TIME: 2024-10-20 17:55:08 -- STEP: 3100/16928 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.03305777907371521 (0.03588733414368285) + | > loss_mel_ce: 4.335571765899658 (4.428765368384718) + | > loss: 1.0921573638916016 (1.1161631756828698) + | > current_lr: 5e-06 + | > step_time: 0.2701 (0.3196465604535994) + | > loader_time: 0.0084 (0.010321390474996276) + + + --> TIME: 2024-10-20 17:55:35 -- STEP: 3150/16928 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.03760598227381706 (0.035878999765430214) + | > loss_mel_ce: 4.153406143188477 (4.425430097201531) + | > loss: 1.0477529764175415 (1.1153272742127631) + | > current_lr: 5e-06 + | > step_time: 0.4989 (0.3197492678960163) + | > loader_time: 0.0114 (0.010313304643782377) + + + --> TIME: 2024-10-20 17:56:01 -- STEP: 3200/16928 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.03571788966655731 (0.035882317991927294) + | > loss_mel_ce: 3.9891467094421387 (4.422762044221178) + | > loss: 1.0062161684036255 (1.114661090597512) + | > current_lr: 5e-06 + | > step_time: 0.2709 (0.3194376544654371) + | > loader_time: 0.009 (0.01030540622770788) + + + --> TIME: 2024-10-20 17:56:27 -- STEP: 3250/16928 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.03438733518123627 (0.03586828091912542) + | > loss_mel_ce: 4.256091117858887 (4.420470825121966) + | > loss: 1.0726195573806763 (1.1140847765299016) + | > current_lr: 5e-06 + | > step_time: 0.3097 (0.31940529214418867) + | > loader_time: 0.0088 (0.010292943661029539) + + + --> TIME: 2024-10-20 17:56:53 -- STEP: 3300/16928 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.03644707426428795 (0.03586360150383727) + | > loss_mel_ce: 4.496747016906738 (4.416328159823576) + | > loss: 1.1332985162734985 (1.1130479403156253) + | > current_lr: 5e-06 + | > step_time: 0.2878 (0.319284406864282) + | > loader_time: 0.0093 (0.010275808464397101) + + + --> TIME: 2024-10-20 17:57:20 -- STEP: 3350/16928 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.03483361750841141 (0.0358523098290411) + | > loss_mel_ce: 4.044952392578125 (4.4150195613548116) + | > loss: 1.0199464559555054 (1.112717967869633) + | > current_lr: 5e-06 + | > step_time: 0.3408 (0.31918975879896944) + | > loader_time: 0.0091 (0.010261865373867678) + + + --> TIME: 2024-10-20 17:57:46 -- STEP: 3400/16928 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.0335642471909523 (0.03585017485544079) + | > loss_mel_ce: 4.108895778656006 (4.4129139275410765) + | > loss: 1.0356149673461914 (1.1121910256322722) + | > current_lr: 5e-06 + | > step_time: 0.2542 (0.3191918345759898) + | > loader_time: 0.009 (0.01024604944621817) + + + --> TIME: 2024-10-20 17:58:13 -- STEP: 3450/16928 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.033792879432439804 (0.03584532347656241) + | > loss_mel_ce: 4.249755859375 (4.409625848825446) + | > loss: 1.0708872079849243 (1.1113677930313635) + | > current_lr: 5e-06 + | > step_time: 0.3961 (0.3190847026437954) + | > loader_time: 0.0091 (0.010232940203901668) + + + --> TIME: 2024-10-20 17:58:39 -- STEP: 3500/16928 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.03795192390680313 (0.03584228168268282) + | > loss_mel_ce: 4.778153419494629 (4.40770989819937) + | > loss: 1.2040263414382935 (1.11088804481711) + | > current_lr: 5e-06 + | > step_time: 0.2222 (0.3190611325672696) + | > loader_time: 0.0085 (0.010219272545405812) + + + --> TIME: 2024-10-20 17:59:06 -- STEP: 3550/16928 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.037466954439878464 (0.03583213709294787) + | > loss_mel_ce: 3.848914623260498 (4.403640527187957) + | > loss: 0.9715954065322876 (1.1098681659261982) + | > current_lr: 5e-06 + | > step_time: 0.3478 (0.31889930389296856) + | > loader_time: 0.0096 (0.010206173238619968) + + + --> TIME: 2024-10-20 17:59:32 -- STEP: 3600/16928 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.03527304530143738 (0.03582487875078279) + | > loss_mel_ce: 4.589357376098633 (4.400841508507754) + | > loss: 1.1561576128005981 (1.109166596763667) + | > current_lr: 5e-06 + | > step_time: 0.3477 (0.31887798534499284) + | > loader_time: 0.0093 (0.010197281771236012) + + + --> TIME: 2024-10-20 18:00:00 -- STEP: 3650/16928 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.034891873598098755 (0.03582081646849839) + | > loss_mel_ce: 4.26510763168335 (4.398136328475139) + | > loss: 1.0749999284744263 (1.1084892860667377) + | > current_lr: 5e-06 + | > step_time: 0.2488 (0.318989538950463) + | > loader_time: 0.0087 (0.010187272829552235) + + + --> TIME: 2024-10-20 18:00:28 -- STEP: 3700/16928 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.03395090252161026 (0.03582274040358284) + | > loss_mel_ce: 3.6767361164093018 (4.395914643004154) + | > loss: 0.9276717305183411 (1.1079343456996493) + | > current_lr: 5e-06 + | > step_time: 0.2241 (0.3191787568942922) + | > loader_time: 0.009 (0.01017627097464898) + + + --> TIME: 2024-10-20 18:00:54 -- STEP: 3750/16928 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.03602013364434242 (0.03582448555280757) + | > loss_mel_ce: 4.589614391326904 (4.3942164990107395) + | > loss: 1.156408667564392 (1.107510246022546) + | > current_lr: 5e-06 + | > step_time: 0.3627 (0.3192423048655193) + | > loader_time: 0.0096 (0.010166358375549332) + + + --> TIME: 2024-10-20 18:01:23 -- STEP: 3800/16928 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.035952724516391754 (0.035822170978706054) + | > loss_mel_ce: 4.247427463531494 (4.393345926247163) + | > loss: 1.0708450078964233 (1.1072920242579398) + | > current_lr: 5e-06 + | > step_time: 0.3764 (0.3195821577624272) + | > loader_time: 0.0096 (0.010157487643392478) + + + --> TIME: 2024-10-20 18:01:50 -- STEP: 3850/16928 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.03349754586815834 (0.03581844509712279) + | > loss_mel_ce: 4.401259422302246 (4.391007899371077) + | > loss: 1.1086891889572144 (1.1067065860698788) + | > current_lr: 5e-06 + | > step_time: 0.2331 (0.3195990800238276) + | > loader_time: 0.0092 (0.010151383288494967) + + + --> TIME: 2024-10-20 18:02:18 -- STEP: 3900/16928 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.0346049927175045 (0.03581811183729225) + | > loss_mel_ce: 4.0535125732421875 (4.388451450726942) + | > loss: 1.0220293998718262 (1.1060673906252971) + | > current_lr: 5e-06 + | > step_time: 0.2464 (0.3198003759750954) + | > loader_time: 0.0082 (0.01014035243254443) + + + --> TIME: 2024-10-20 18:02:44 -- STEP: 3950/16928 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.03679783269762993 (0.035823071729155956) + | > loss_mel_ce: 4.325137615203857 (4.385976729936254) + | > loss: 1.0904839038848877 (1.1054499504385151) + | > current_lr: 5e-06 + | > step_time: 0.4691 (0.3198082387900052) + | > loader_time: 0.0096 (0.01012732638588436) + + + --> TIME: 2024-10-20 18:03:10 -- STEP: 4000/16928 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.039123788475990295 (0.035826752572320326) + | > loss_mel_ce: 4.1140031814575195 (4.382819220304506) + | > loss: 1.038281798362732 (1.1046614932119883) + | > current_lr: 5e-06 + | > step_time: 0.2498 (0.3195424287915231) + | > loader_time: 0.0091 (0.010111669957637801) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_4000.pth + + --> TIME: 2024-10-20 18:03:51 -- STEP: 4050/16928 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.035427406430244446 (0.0358252106735735) + | > loss_mel_ce: 4.549306869506836 (4.3808142741521365) + | > loss: 1.1461836099624634 (1.1041598713250844) + | > current_lr: 5e-06 + | > step_time: 0.304 (0.31948104257936843) + | > loader_time: 0.0087 (0.010102442223348749) + + + --> TIME: 2024-10-20 18:04:16 -- STEP: 4100/16928 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.03668249398469925 (0.03581673425554131) + | > loss_mel_ce: 3.991861343383789 (4.378903189403271) + | > loss: 1.0071359872817993 (1.1036799810717781) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.31926359217341366) + | > loader_time: 0.0092 (0.01008848475246896) + + + --> TIME: 2024-10-20 18:04:44 -- STEP: 4150/16928 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.035751551389694214 (0.035821969160383646) + | > loss_mel_ce: 4.591662406921387 (4.3779176935517805) + | > loss: 1.156853437423706 (1.103434915772407) + | > current_lr: 5e-06 + | > step_time: 0.5082 (0.319339808153819) + | > loader_time: 0.014 (0.010076853912996972) + + + --> TIME: 2024-10-20 18:05:09 -- STEP: 4200/16928 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.03226400539278984 (0.03581950307468921) + | > loss_mel_ce: 4.220775604248047 (4.375653344392792) + | > loss: 1.0632599592208862 (1.1028682119221946) + | > current_lr: 5e-06 + | > step_time: 0.2519 (0.3191400125480834) + | > loader_time: 0.0104 (0.010065553245090316) + + + --> TIME: 2024-10-20 18:05:34 -- STEP: 4250/16928 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.036666639149188995 (0.03582042660914792) + | > loss_mel_ce: 4.118873119354248 (4.37349424558529) + | > loss: 1.0388849973678589 (1.1023286682437479) + | > current_lr: 5e-06 + | > step_time: 0.3126 (0.318927714291741) + | > loader_time: 0.0089 (0.010053903074825525) + + + --> TIME: 2024-10-20 18:06:01 -- STEP: 4300/16928 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.03588692471385002 (0.03580913167576796) + | > loss_mel_ce: 4.252762794494629 (4.370974837791104) + | > loss: 1.072162389755249 (1.101695992489197) + | > current_lr: 5e-06 + | > step_time: 0.1743 (0.3189442895179572) + | > loader_time: 0.0087 (0.010042596972265925) + + + --> TIME: 2024-10-20 18:06:27 -- STEP: 4350/16928 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.03507723659276962 (0.035807375779867784) + | > loss_mel_ce: 4.553157806396484 (4.368245087656498) + | > loss: 1.1470587253570557 (1.101013115973311) + | > current_lr: 5e-06 + | > step_time: 0.4922 (0.3188283073491064) + | > loader_time: 0.009 (0.010031162733319178) + + + --> TIME: 2024-10-20 18:06:54 -- STEP: 4400/16928 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.03711877018213272 (0.035810858619568633) + | > loss_mel_ce: 4.360525131225586 (4.366149875792609) + | > loss: 1.0994110107421875 (1.1004901836947987) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.31887054996057) + | > loader_time: 0.0087 (0.010022442611781049) + + + --> TIME: 2024-10-20 18:07:22 -- STEP: 4450/16928 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.04030560329556465 (0.035813352070916175) + | > loss_mel_ce: 4.751277923583984 (4.363852011755628) + | > loss: 1.197895884513855 (1.0999163409967128) + | > current_lr: 5e-06 + | > step_time: 0.4971 (0.31908823249045404) + | > loader_time: 0.0094 (0.010013985580272904) + + + --> TIME: 2024-10-20 18:07:51 -- STEP: 4500/16928 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.036164894700050354 (0.03581295967350397) + | > loss_mel_ce: 4.1247334480285645 (4.361806519296453) + | > loss: 1.040224552154541 (1.0994048698080936) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.3192958418528241) + | > loader_time: 0.009 (0.010005609088473864) + + + --> TIME: 2024-10-20 18:08:17 -- STEP: 4550/16928 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.03637339174747467 (0.03580823833061439) + | > loss_mel_ce: 4.270094871520996 (4.359646844497111) + | > loss: 1.0766171216964722 (1.098863770804565) + | > current_lr: 5e-06 + | > step_time: 0.2478 (0.31924513539115174) + | > loader_time: 0.0087 (0.009996040155599422) + + + --> TIME: 2024-10-20 18:08:44 -- STEP: 4600/16928 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.03513900563120842 (0.0358047282894182) + | > loss_mel_ce: 4.107263088226318 (4.358260953737358) + | > loss: 1.0356005430221558 (1.0985164205794768) + | > current_lr: 5e-06 + | > step_time: 0.3474 (0.31934217463368975) + | > loader_time: 0.0093 (0.009986513231111629) + + + --> TIME: 2024-10-20 18:09:12 -- STEP: 4650/16928 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.035879578441381454 (0.035806730026679584) + | > loss_mel_ce: 4.230328559875488 (4.356140152818431) + | > loss: 1.0665520429611206 (1.097986720787583) + | > current_lr: 5e-06 + | > step_time: 0.3587 (0.3193976113104054) + | > loader_time: 0.0092 (0.009977422119468792) + + + --> TIME: 2024-10-20 18:09:39 -- STEP: 4700/16928 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.036136046051979065 (0.03580611883087987) + | > loss_mel_ce: 3.9481585025787354 (4.354502432143454) + | > loss: 0.9960736632347107 (1.0975771379090387) + | > current_lr: 5e-06 + | > step_time: 0.3024 (0.31946156273496934) + | > loader_time: 0.009 (0.00996856988744535) + + + --> TIME: 2024-10-20 18:10:06 -- STEP: 4750/16928 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.03485588729381561 (0.03580376066738044) + | > loss_mel_ce: 3.956345319747925 (4.352970897975738) + | > loss: 0.9978002905845642 (1.0971936648895875) + | > current_lr: 5e-06 + | > step_time: 0.4638 (0.3195001045026279) + | > loader_time: 0.0092 (0.009960480338648766) + + + --> TIME: 2024-10-20 18:10:33 -- STEP: 4800/16928 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.03433024138212204 (0.03579770509153597) + | > loss_mel_ce: 3.76126766204834 (4.350793164720155) + | > loss: 0.9488994479179382 (1.0966477177540468) + | > current_lr: 5e-06 + | > step_time: 0.2054 (0.31963060503204715) + | > loader_time: 0.0092 (0.009956269065539066) + + + --> TIME: 2024-10-20 18:11:03 -- STEP: 4850/16928 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.03369259834289551 (0.0357933247185398) + | > loss_mel_ce: 3.827042579650879 (4.349737886006086) + | > loss: 0.9651837944984436 (1.0963828029583422) + | > current_lr: 5e-06 + | > step_time: 0.3143 (0.3200351068654017) + | > loader_time: 0.0092 (0.009950149526301143) + + + --> TIME: 2024-10-20 18:11:30 -- STEP: 4900/16928 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.03576182574033737 (0.03579051201858988) + | > loss_mel_ce: 3.9864649772644043 (4.348044206220295) + | > loss: 1.0055567026138306 (1.0959586798658192) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.3200312667476894) + | > loader_time: 0.0131 (0.009944612201379294) + + + --> TIME: 2024-10-20 18:11:56 -- STEP: 4950/16928 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.03671840578317642 (0.03578845678104287) + | > loss_mel_ce: 4.122593879699707 (4.346016928836569) + | > loss: 1.0398280620574951 (1.0954513466599027) + | > current_lr: 5e-06 + | > step_time: 0.3379 (0.31988021436363806) + | > loader_time: 0.0092 (0.009937047380389611) + + + --> TIME: 2024-10-20 18:12:22 -- STEP: 5000/16928 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.036639079451560974 (0.03578743918463584) + | > loss_mel_ce: 4.15232515335083 (4.344029604911819) + | > loss: 1.0472410917282104 (1.09495426130295) + | > current_lr: 5e-06 + | > step_time: 0.3224 (0.31984677700996406) + | > loader_time: 0.0095 (0.009930716228485131) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_5000.pth + + --> TIME: 2024-10-20 18:13:03 -- STEP: 5050/16928 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.037810251116752625 (0.03578928121688339) + | > loss_mel_ce: 4.083659648895264 (4.342636195551068) + | > loss: 1.0303674936294556 (1.094606369450543) + | > current_lr: 5e-06 + | > step_time: 0.2598 (0.3200494364464638) + | > loader_time: 0.0096 (0.009930434463047762) + + + --> TIME: 2024-10-20 18:13:32 -- STEP: 5100/16928 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.03613004460930824 (0.03578871905292366) + | > loss_mel_ce: 4.306159496307373 (4.340345492269498) + | > loss: 1.085572361946106 (1.0940335530393288) + | > current_lr: 5e-06 + | > step_time: 0.3999 (0.3203161096572878) + | > loader_time: 0.0099 (0.00992450845007805) + + + --> TIME: 2024-10-20 18:13:58 -- STEP: 5150/16928 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.033616553992033005 (0.03578213791679407) + | > loss_mel_ce: 4.087193965911865 (4.338239737899574) + | > loss: 1.0302026271820068 (1.09350546920184) + | > current_lr: 5e-06 + | > step_time: 0.4465 (0.3202521328787206) + | > loader_time: 0.0099 (0.009916757657689979) + + + --> TIME: 2024-10-20 18:14:25 -- STEP: 5200/16928 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.03677105903625488 (0.03577748219231858) + | > loss_mel_ce: 3.8323919773101807 (4.336388093829169) + | > loss: 0.9672907590866089 (1.093041394192441) + | > current_lr: 5e-06 + | > step_time: 0.1874 (0.32023917766717797) + | > loader_time: 0.0086 (0.009911548449442958) + + + --> TIME: 2024-10-20 18:14:51 -- STEP: 5250/16928 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.03684602305293083 (0.03577311272386989) + | > loss_mel_ce: 4.389496326446533 (4.33416755753473) + | > loss: 1.1065856218338013 (1.092485167798545) + | > current_lr: 5e-06 + | > step_time: 0.2088 (0.32023249866848946) + | > loader_time: 0.009 (0.00990608478727797) + + + --> TIME: 2024-10-20 18:15:20 -- STEP: 5300/16928 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.03401046246290207 (0.0357652951703178) + | > loss_mel_ce: 3.809840440750122 (4.332244542184881) + | > loss: 0.96096271276474 (1.0920024595395603) + | > current_lr: 5e-06 + | > step_time: 0.2232 (0.3204367652929059) + | > loader_time: 0.0116 (0.009899974049262295) + + + --> TIME: 2024-10-20 18:15:47 -- STEP: 5350/16928 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.03510415926575661 (0.035760839526828694) + | > loss_mel_ce: 3.8357691764831543 (4.329714162773081) + | > loss: 0.9677183628082275 (1.0913687507237249) + | > current_lr: 5e-06 + | > step_time: 0.2512 (0.3204763072896233) + | > loader_time: 0.0089 (0.009893399666402954) + + + --> TIME: 2024-10-20 18:16:13 -- STEP: 5400/16928 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.035322900861501694 (0.03575746473166399) + | > loss_mel_ce: 4.10865592956543 (4.328215797786379) + | > loss: 1.0359946489334106 (1.0909933157651537) + | > current_lr: 5e-06 + | > step_time: 0.248 (0.320312001749322) + | > loader_time: 0.009 (0.009885706813247123) + + + --> TIME: 2024-10-20 18:16:39 -- STEP: 5450/16928 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.03452460095286369 (0.035753027138838496) + | > loss_mel_ce: 4.24363374710083 (4.325966987259909) + | > loss: 1.0695395469665527 (1.090430003721781) + | > current_lr: 5e-06 + | > step_time: 0.3776 (0.32019375836083724) + | > loader_time: 0.0099 (0.009878448083860078) + + + --> TIME: 2024-10-20 18:17:05 -- STEP: 5500/16928 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.031029509380459785 (0.03574627295305777) + | > loss_mel_ce: 3.6851468086242676 (4.323775225422619) + | > loss: 0.9290440678596497 (1.0898803747242156) + | > current_lr: 5e-06 + | > step_time: 0.1553 (0.3201445210196762) + | > loader_time: 0.0087 (0.009872702165083461) + + + --> TIME: 2024-10-20 18:17:31 -- STEP: 5550/16928 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.033178217709064484 (0.03574186500531054) + | > loss_mel_ce: 4.062270164489746 (4.322306941264406) + | > loss: 1.0238621234893799 (1.0895122016872374) + | > current_lr: 5e-06 + | > step_time: 0.3784 (0.3200786470292931) + | > loader_time: 0.0095 (0.009866644111839509) + + + --> TIME: 2024-10-20 18:17:57 -- STEP: 5600/16928 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.030115408822894096 (0.035738553247481035) + | > loss_mel_ce: 3.9615073204040527 (4.320198917133491) + | > loss: 0.9979056715965271 (1.0889843677516495) + | > current_lr: 5e-06 + | > step_time: 0.183 (0.3199498832225807) + | > loader_time: 0.0087 (0.009860798375947142) + + + --> TIME: 2024-10-20 18:18:23 -- STEP: 5650/16928 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.03384269028902054 (0.035736104504460804) + | > loss_mel_ce: 4.175619125366211 (4.3187771213160415) + | > loss: 1.0523654222488403 (1.0886283065681972) + | > current_lr: 5e-06 + | > step_time: 0.4949 (0.3198057140080284) + | > loader_time: 0.0094 (0.00985344312887277) + + + --> TIME: 2024-10-20 18:18:48 -- STEP: 5700/16928 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.03508566692471504 (0.03572996411113099) + | > loss_mel_ce: 4.45463228225708 (4.316059114180143) + | > loss: 1.1224294900894165 (1.087947269678117) + | > current_lr: 5e-06 + | > step_time: 0.2532 (0.31962476253509614) + | > loader_time: 0.0093 (0.009847639694548504) + + + --> TIME: 2024-10-20 18:19:16 -- STEP: 5750/16928 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.03331828862428665 (0.03572803043347335) + | > loss_mel_ce: 4.051603317260742 (4.314033571201847) + | > loss: 1.0212304592132568 (1.0874404004760436) + | > current_lr: 5e-06 + | > step_time: 0.2513 (0.31972370931376676) + | > loader_time: 0.0092 (0.00984277766683828) + + + --> TIME: 2024-10-20 18:19:43 -- STEP: 5800/16928 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.03289736807346344 (0.03572429381471507) + | > loss_mel_ce: 4.632205009460449 (4.312762274906581) + | > loss: 1.1662756204605103 (1.0871216422730514) + | > current_lr: 5e-06 + | > step_time: 0.2386 (0.31981538151872657) + | > loader_time: 0.0092 (0.009837961895712496) + + + --> TIME: 2024-10-20 18:20:11 -- STEP: 5850/16928 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.037729501724243164 (0.03572033410525724) + | > loss_mel_ce: 4.095391750335693 (4.310964264299145) + | > loss: 1.033280372619629 (1.0866711496899308) + | > current_lr: 5e-06 + | > step_time: 0.4682 (0.3199226268540091) + | > loader_time: 0.0095 (0.009834292362897826) + + + --> TIME: 2024-10-20 18:20:37 -- STEP: 5900/16928 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.03484288230538368 (0.035721988297241145) + | > loss_mel_ce: 4.477766513824463 (4.3092484492771) + | > loss: 1.1281523704528809 (1.0862426094988684) + | > current_lr: 5e-06 + | > step_time: 0.1886 (0.31982280416003833) + | > loader_time: 0.009 (0.009831199605586169) + + + --> TIME: 2024-10-20 18:21:04 -- STEP: 5950/16928 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.035479165613651276 (0.03571979127761689) + | > loss_mel_ce: 4.346982955932617 (4.308027556162953) + | > loss: 1.0956155061721802 (1.08593683696595) + | > current_lr: 5e-06 + | > step_time: 0.2977 (0.31984786498446494) + | > loader_time: 0.0099 (0.009826226154295344) + + + --> TIME: 2024-10-20 18:21:30 -- STEP: 6000/16928 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.03279389068484306 (0.0357183730897182) + | > loss_mel_ce: 4.835048198699951 (4.306188656131458) + | > loss: 1.2169605493545532 (1.0854767574667963) + | > current_lr: 5e-06 + | > step_time: 0.2218 (0.3197518780628848) + | > loader_time: 0.0087 (0.009820930997530621) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_6000.pth + + --> TIME: 2024-10-20 18:22:11 -- STEP: 6050/16928 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.03921288624405861 (0.03571535353294816) + | > loss_mel_ce: 4.3747782707214355 (4.30516427323841) + | > loss: 1.1034977436065674 (1.0852199068345354) + | > current_lr: 5e-06 + | > step_time: 0.3355 (0.3196730017465017) + | > loader_time: 0.0088 (0.009818529570398252) + + + --> TIME: 2024-10-20 18:22:36 -- STEP: 6100/16928 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.0338808037340641 (0.035715837613603016) + | > loss_mel_ce: 3.675365686416626 (4.303409316617964) + | > loss: 0.9273115992546082 (1.0847812887293433) + | > current_lr: 5e-06 + | > step_time: 0.4006 (0.3195321266377571) + | > loader_time: 0.0095 (0.009812369463873689) + + + --> TIME: 2024-10-20 18:23:02 -- STEP: 6150/16928 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.03401874005794525 (0.035716840003927386) + | > loss_mel_ce: 4.247733116149902 (4.3026596094922605) + | > loss: 1.0704379081726074 (1.0845941125571288) + | > current_lr: 5e-06 + | > step_time: 0.2613 (0.3194435939168556) + | > loader_time: 0.0101 (0.009808188841594906) + + + --> TIME: 2024-10-20 18:23:28 -- STEP: 6200/16928 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.03409641608595848 (0.03571214187289424) + | > loss_mel_ce: 3.6862716674804688 (4.3001310646534225) + | > loss: 0.9300920367240906 (1.0839608017667695) + | > current_lr: 5e-06 + | > step_time: 0.3442 (0.3192977247315082) + | > loader_time: 0.0092 (0.009803517441595744) + + + --> TIME: 2024-10-20 18:23:56 -- STEP: 6250/16928 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.035192232578992844 (0.0357098974639176) + | > loss_mel_ce: 4.2567315101623535 (4.298506247100855) + | > loss: 1.0729808807373047 (1.0835540363216452) + | > current_lr: 5e-06 + | > step_time: 0.1898 (0.31951998130798487) + | > loader_time: 0.0086 (0.009798187179565418) + + + --> TIME: 2024-10-20 18:24:22 -- STEP: 6300/16928 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.03769128769636154 (0.03571170024455527) + | > loss_mel_ce: 4.127147674560547 (4.297316118308498) + | > loss: 1.0412096977233887 (1.0832569548251176) + | > current_lr: 5e-06 + | > step_time: 0.2758 (0.3194448978181883) + | > loader_time: 0.009 (0.009793829766530827) + + + --> TIME: 2024-10-20 18:24:49 -- STEP: 6350/16928 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.03313494473695755 (0.03570870405195015) + | > loss_mel_ce: 3.8488781452178955 (4.2953061139489925) + | > loss: 0.9705032706260681 (1.0827537046830522) + | > current_lr: 5e-06 + | > step_time: 0.2173 (0.3194328548776836) + | > loader_time: 0.0082 (0.00978860220571201) + + + --> TIME: 2024-10-20 18:25:17 -- STEP: 6400/16928 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.037911344319581985 (0.03570992232154804) + | > loss_mel_ce: 3.809556007385254 (4.293645414486547) + | > loss: 0.9618668556213379 (1.082338834339756) + | > current_lr: 5e-06 + | > step_time: 0.208 (0.3194865787774339) + | > loader_time: 0.0089 (0.00978506777435539) + + + --> TIME: 2024-10-20 18:25:43 -- STEP: 6450/16928 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.034783199429512024 (0.03570538311359247) + | > loss_mel_ce: 4.649790287017822 (4.292009852357632) + | > loss: 1.1711434125900269 (1.0819288089940746) + | > current_lr: 5e-06 + | > step_time: 0.3591 (0.31945995914843794) + | > loader_time: 0.0099 (0.009781490370284661) + + + --> TIME: 2024-10-20 18:26:09 -- STEP: 6500/16928 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.0316133126616478 (0.03569930392704313) + | > loss_mel_ce: 4.244680881500244 (4.29052749556763) + | > loss: 1.0690735578536987 (1.0815566999728896) + | > current_lr: 5e-06 + | > step_time: 0.2485 (0.3194279771217947) + | > loader_time: 0.0087 (0.009777481445899355) + + + --> TIME: 2024-10-20 18:26:37 -- STEP: 6550/16928 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.03341655060648918 (0.03569222711026651) + | > loss_mel_ce: 4.174530982971191 (4.289061839307549) + | > loss: 1.0519869327545166 (1.081188516762424) + | > current_lr: 5e-06 + | > step_time: 0.3005 (0.31947994406896835) + | > loader_time: 0.0087 (0.009773860523718887) + + + --> TIME: 2024-10-20 18:27:03 -- STEP: 6600/16928 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.03698189556598663 (0.0356898101972358) + | > loss_mel_ce: 4.299375534057617 (4.287930899136008) + | > loss: 1.0840893983840942 (1.080905177520987) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.319407174406631) + | > loader_time: 0.0089 (0.009770706241781033) + + + --> TIME: 2024-10-20 18:27:28 -- STEP: 6650/16928 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.03551316633820534 (0.03569218268155811) + | > loss_mel_ce: 4.233281135559082 (4.286420668874486) + | > loss: 1.067198634147644 (1.080528213027728) + | > current_lr: 5e-06 + | > step_time: 0.4437 (0.31930366835199936) + | > loader_time: 0.0103 (0.00976592490547579) + + + --> TIME: 2024-10-20 18:27:56 -- STEP: 6700/16928 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.041187457740306854 (0.035692220287703136) + | > loss_mel_ce: 3.9087929725646973 (4.285031803935338) + | > loss: 0.9874951243400574 (1.0801810061664752) + | > current_lr: 5e-06 + | > step_time: 0.399 (0.31934527742329066) + | > loader_time: 0.0108 (0.009761725468422022) + + + --> TIME: 2024-10-20 18:28:23 -- STEP: 6750/16928 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.03320834040641785 (0.03569271173052196) + | > loss_mel_ce: 4.446427822113037 (4.283078586437098) + | > loss: 1.1199090480804443 (1.0796928246816024) + | > current_lr: 5e-06 + | > step_time: 0.38 (0.3193568982371591) + | > loader_time: 0.0092 (0.00975847767017503) + + + --> TIME: 2024-10-20 18:28:50 -- STEP: 6800/16928 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.034003905951976776 (0.0356864373574908) + | > loss_mel_ce: 4.195446014404297 (4.2815432579727855) + | > loss: 1.05736243724823 (1.079307423970282) + | > current_lr: 5e-06 + | > step_time: 0.3229 (0.31935041294378524) + | > loader_time: 0.0098 (0.009753834640278508) + + + --> TIME: 2024-10-20 18:29:16 -- STEP: 6850/16928 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.03457259759306908 (0.03568148259671701) + | > loss_mel_ce: 4.316437721252441 (4.280183527313019) + | > loss: 1.0877525806427002 (1.0789662526210761) + | > current_lr: 5e-06 + | > step_time: 0.3345 (0.3192842894226979) + | > loader_time: 0.0092 (0.009749781789570802) + + + --> TIME: 2024-10-20 18:29:44 -- STEP: 6900/16928 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.03752901777625084 (0.03567765136937715) + | > loss_mel_ce: 3.7368404865264893 (4.278553269773301) + | > loss: 0.943592369556427 (1.0785577303948637) + | > current_lr: 5e-06 + | > step_time: 0.3508 (0.3194366372495471) + | > loader_time: 0.009 (0.009747432038403912) + + + --> TIME: 2024-10-20 18:30:11 -- STEP: 6950/16928 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.03891957178711891 (0.03567475869956829) + | > loss_mel_ce: 4.002504348754883 (4.277271407422415) + | > loss: 1.0103559494018555 (1.078236541585102) + | > current_lr: 5e-06 + | > step_time: 0.2861 (0.31942583053232226) + | > loader_time: 0.0087 (0.009742580386374468) + + + --> TIME: 2024-10-20 18:30:39 -- STEP: 7000/16928 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.0325503908097744 (0.03567601766783192) + | > loss_mel_ce: 4.288796901702881 (4.276210382836219) + | > loss: 1.0803368091583252 (1.0779716002089679) + | > current_lr: 5e-06 + | > step_time: 0.2511 (0.31953787102018216) + | > loader_time: 0.0087 (0.009740572725023516) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_7000.pth + + --> TIME: 2024-10-20 18:31:20 -- STEP: 7050/16928 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.03789713233709335 (0.03567185599806019) + | > loss_mel_ce: 3.941230535507202 (4.274913464133633) + | > loss: 0.9947819113731384 (1.0776463301300199) + | > current_lr: 5e-06 + | > step_time: 0.4398 (0.3195415200240236) + | > loader_time: 0.0092 (0.009741621998184935) + + + --> TIME: 2024-10-20 18:31:47 -- STEP: 7100/16928 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.031012360006570816 (0.03567206083180391) + | > loss_mel_ce: 3.783703565597534 (4.273858991676662) + | > loss: 0.9536789655685425 (1.0773827631960498) + | > current_lr: 5e-06 + | > step_time: 0.1835 (0.31956742347126404) + | > loader_time: 0.0087 (0.009737225351199271) + + + --> TIME: 2024-10-20 18:32:14 -- STEP: 7150/16928 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.03167873993515968 (0.035667712724135686) + | > loss_mel_ce: 3.6859307289123535 (4.272507302477653) + | > loss: 0.9294023513793945 (1.077043753879059) + | > current_lr: 5e-06 + | > step_time: 0.216 (0.3195764221844986) + | > loader_time: 0.0085 (0.009733473504339873) + + + --> TIME: 2024-10-20 18:32:40 -- STEP: 7200/16928 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.0325649119913578 (0.0356643749657085) + | > loss_mel_ce: 4.131974697113037 (4.2710005584690425) + | > loss: 1.0411349534988403 (1.076666233440245) + | > current_lr: 5e-06 + | > step_time: 0.2722 (0.3194462552666677) + | > loader_time: 0.0098 (0.009729636112848892) + + + --> TIME: 2024-10-20 18:33:07 -- STEP: 7250/16928 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.03201307728886604 (0.03566274998696693) + | > loss_mel_ce: 3.8403031826019287 (4.270392450365538) + | > loss: 0.9680790901184082 (1.0765138001853063) + | > current_lr: 5e-06 + | > step_time: 0.1846 (0.3195238445216224) + | > loader_time: 0.0083 (0.009725276815480174) + + + --> TIME: 2024-10-20 18:33:35 -- STEP: 7300/16928 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.04021124169230461 (0.03566224001832796) + | > loss_mel_ce: 4.292073726654053 (4.269420841262779) + | > loss: 1.083071231842041 (1.076270770422404) + | > current_lr: 5e-06 + | > step_time: 0.1511 (0.31960005858173235) + | > loader_time: 0.009 (0.00972136010862374) + + + --> TIME: 2024-10-20 18:34:01 -- STEP: 7350/16928 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.034474413841962814 (0.03566033409312854) + | > loss_mel_ce: 4.225036144256592 (4.26820608628852) + | > loss: 1.0648776292800903 (1.0759666051588943) + | > current_lr: 5e-06 + | > step_time: 0.2678 (0.3195770442404727) + | > loader_time: 0.0089 (0.009717987021621367) + + + --> TIME: 2024-10-20 18:34:28 -- STEP: 7400/16928 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.03432301804423332 (0.03565712751619297) + | > loss_mel_ce: 4.1314849853515625 (4.267014770636703) + | > loss: 1.0414520502090454 (1.0756679746266993) + | > current_lr: 5e-06 + | > step_time: 0.357 (0.3195625451771001) + | > loader_time: 0.0092 (0.009713360425588222) + + + --> TIME: 2024-10-20 18:34:55 -- STEP: 7450/16928 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.0350053645670414 (0.03565466327955239) + | > loss_mel_ce: 4.140570163726807 (4.266060689055695) + | > loss: 1.0438939332962036 (1.0754288382018204) + | > current_lr: 5e-06 + | > step_time: 0.3366 (0.31958642921191854) + | > loader_time: 0.0087 (0.009709561111142946) + + + --> TIME: 2024-10-20 18:35:22 -- STEP: 7500/16928 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.03816454857587814 (0.03565115737169961) + | > loss_mel_ce: 4.077631950378418 (4.265094118944816) + | > loss: 1.0289491415023804 (1.0751863191843065) + | > current_lr: 5e-06 + | > step_time: 0.2003 (0.31957209574381634) + | > loader_time: 0.0084 (0.009706694285074846) + + + --> TIME: 2024-10-20 18:35:48 -- STEP: 7550/16928 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.03492163494229317 (0.035648817462192665) + | > loss_mel_ce: 4.228000164031982 (4.263480546111318) + | > loss: 1.0657304525375366 (1.0747823410160526) + | > current_lr: 5e-06 + | > step_time: 0.2719 (0.31956296472360085) + | > loader_time: 0.0078 (0.00970151370724303) + + + --> TIME: 2024-10-20 18:36:14 -- STEP: 7600/16928 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.03531540930271149 (0.035649278721722924) + | > loss_mel_ce: 3.8926942348480225 (4.262132521993253) + | > loss: 0.9820024371147156 (1.07444545032947) + | > current_lr: 5e-06 + | > step_time: 0.2522 (0.31946433252410006) + | > loader_time: 0.0087 (0.0096971545721355) + + + --> TIME: 2024-10-20 18:36:41 -- STEP: 7650/16928 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.03535839542746544 (0.03564913692330204) + | > loss_mel_ce: 3.9688162803649902 (4.261027819658434) + | > loss: 1.001043677330017 (1.0741692393122184) + | > current_lr: 5e-06 + | > step_time: 0.4913 (0.3194971890231375) + | > loader_time: 0.0099 (0.009693100031684403) + + + --> TIME: 2024-10-20 18:37:07 -- STEP: 7700/16928 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.03966589272022247 (0.035642633242411466) + | > loss_mel_ce: 4.164024353027344 (4.259511409765734) + | > loss: 1.0509225130081177 (1.0737885109325536) + | > current_lr: 5e-06 + | > step_time: 0.2708 (0.3194498549498533) + | > loader_time: 0.0088 (0.009688977978446242) + + + --> TIME: 2024-10-20 18:37:35 -- STEP: 7750/16928 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.03570667654275894 (0.03564262000567471) + | > loss_mel_ce: 3.770333766937256 (4.258494554611964) + | > loss: 0.9515101313591003 (1.0735342938746204) + | > current_lr: 5e-06 + | > step_time: 0.3587 (0.31950758429496634) + | > loader_time: 0.0094 (0.00968543563350552) + + + --> TIME: 2024-10-20 18:38:01 -- STEP: 7800/16928 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.033956900238990784 (0.03563943649880951) + | > loss_mel_ce: 3.8261196613311768 (4.257736928829799) + | > loss: 0.965019166469574 (1.0733440915590706) + | > current_lr: 5e-06 + | > step_time: 0.3573 (0.31945156091298926) + | > loader_time: 0.0087 (0.009681614729074308) + + + --> TIME: 2024-10-20 18:38:27 -- STEP: 7850/16928 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.035822369158267975 (0.03563790471524361) + | > loss_mel_ce: 3.823615074157715 (4.256604848849565) + | > loss: 0.9648593664169312 (1.0730606886383862) + | > current_lr: 5e-06 + | > step_time: 0.5125 (0.31939467181066156) + | > loader_time: 0.0096 (0.0096781656848397) + + + --> TIME: 2024-10-20 18:38:53 -- STEP: 7900/16928 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.034739118069410324 (0.03563649452845488) + | > loss_mel_ce: 3.8875222206115723 (4.255252814232568) + | > loss: 0.9805653095245361 (1.0727223274300397) + | > current_lr: 5e-06 + | > step_time: 0.288 (0.31934839484057964) + | > loader_time: 0.0089 (0.009675074076350708) + + + --> TIME: 2024-10-20 18:39:19 -- STEP: 7950/16928 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.03672211244702339 (0.035634632007505725) + | > loss_mel_ce: 4.4117889404296875 (4.254564834630734) + | > loss: 1.1121277809143066 (1.072549866835279) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.3192547733978667) + | > loader_time: 0.009 (0.009671923739355291) + + + --> TIME: 2024-10-20 18:39:45 -- STEP: 8000/16928 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.038047995418310165 (0.035637146816123094) + | > loss_mel_ce: 3.6608192920684814 (4.253125097244997) + | > loss: 0.9247168302536011 (1.072190561197701) + | > current_lr: 5e-06 + | > step_time: 0.2248 (0.31919488665461654) + | > loader_time: 0.0086 (0.009667878091335274) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_8000.pth + + --> TIME: 2024-10-20 18:40:25 -- STEP: 8050/16928 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.03142429143190384 (0.03563520771792568) + | > loss_mel_ce: 4.335765838623047 (4.251796007304477) + | > loss: 1.0917975902557373 (1.071857803963729) + | > current_lr: 5e-06 + | > step_time: 0.2563 (0.3191655589334724) + | > loader_time: 0.0087 (0.009666600286590363) + + + --> TIME: 2024-10-20 18:40:51 -- STEP: 8100/16928 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.03682070970535278 (0.035634537164903536) + | > loss_mel_ce: 4.754121780395508 (4.25052242114222) + | > loss: 1.1977356672286987 (1.071539239802481) + | > current_lr: 5e-06 + | > step_time: 0.2718 (0.3191064347455542) + | > loader_time: 0.0085 (0.009663419399732403) + + + --> TIME: 2024-10-20 18:41:19 -- STEP: 8150/16928 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.03549870476126671 (0.03563390904271877) + | > loss_mel_ce: 4.228590488433838 (4.249036831007432) + | > loss: 1.066022276878357 (1.0711676852322822) + | > current_lr: 5e-06 + | > step_time: 0.338 (0.3191407112987506) + | > loader_time: 0.0087 (0.009660153330469401) + + + --> TIME: 2024-10-20 18:41:44 -- STEP: 8200/16928 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.03165462613105774 (0.035632345088098656) + | > loss_mel_ce: 4.7807393074035645 (4.24815166202989) + | > loss: 1.2030985355377197 (1.0709460020428778) + | > current_lr: 5e-06 + | > step_time: 0.287 (0.3190264628573173) + | > loader_time: 0.009 (0.009655121855619455) + + + --> TIME: 2024-10-20 18:42:11 -- STEP: 8250/16928 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.029909232631325722 (0.03563119660385612) + | > loss_mel_ce: 4.092475414276123 (4.246875265323772) + | > loss: 1.030596137046814 (1.0706266157265891) + | > current_lr: 5e-06 + | > step_time: 0.2756 (0.3190513757936893) + | > loader_time: 0.0091 (0.009650897950837085) + + + --> TIME: 2024-10-20 18:42:37 -- STEP: 8300/16928 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.03251459449529648 (0.03562397934047555) + | > loss_mel_ce: 4.488814830780029 (4.245600443977925) + | > loss: 1.130332350730896 (1.0703061060661319) + | > current_lr: 5e-06 + | > step_time: 0.2745 (0.3189665682919065) + | > loader_time: 0.0089 (0.009645395365106032) + + + --> TIME: 2024-10-20 18:43:04 -- STEP: 8350/16928 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.03455866128206253 (0.035623138241335825) + | > loss_mel_ce: 3.6529080867767334 (4.244116913772646) + | > loss: 0.9218667149543762 (1.0699350132485377) + | > current_lr: 5e-06 + | > step_time: 0.1891 (0.31901800872323155) + | > loader_time: 0.0082 (0.00964255895443303) + + + --> TIME: 2024-10-20 18:43:31 -- STEP: 8400/16928 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.03800984099507332 (0.03562025344021422) + | > loss_mel_ce: 4.239778995513916 (4.24258875114579) + | > loss: 1.0694471597671509 (1.0695522513914704) + | > current_lr: 5e-06 + | > step_time: 0.347 (0.3189663836501904) + | > loader_time: 0.0088 (0.00963799635569252) + + + --> TIME: 2024-10-20 18:43:58 -- STEP: 8450/16928 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.03846349939703941 (0.035618855103894266) + | > loss_mel_ce: 4.140029430389404 (4.241472127705653) + | > loss: 1.0446232557296753 (1.0692727459817268) + | > current_lr: 5e-06 + | > step_time: 0.255 (0.31902589673826787) + | > loader_time: 0.0086 (0.00963312730281308) + + + --> TIME: 2024-10-20 18:44:24 -- STEP: 8500/16928 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.03660721704363823 (0.03561587284943618) + | > loss_mel_ce: 4.449456691741943 (4.240320560455339) + | > loss: 1.1215159893035889 (1.0689841085812646) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.3190094251071716) + | > loader_time: 0.0089 (0.009629042036393084) + + + --> TIME: 2024-10-20 18:44:51 -- STEP: 8550/16928 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.03537840023636818 (0.03561151502991609) + | > loss_mel_ce: 3.9838130474090576 (4.23948098681825) + | > loss: 1.0047978162765503 (1.0687731257098483) + | > current_lr: 5e-06 + | > step_time: 0.4305 (0.3189717406278476) + | > loader_time: 0.0096 (0.00962555723580698) + + + --> TIME: 2024-10-20 18:45:19 -- STEP: 8600/16928 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.03791746869683266 (0.03561094714978386) + | > loss_mel_ce: 4.0298752784729 (4.238430238546342) + | > loss: 1.0169482231140137 (1.0685102966497135) + | > current_lr: 5e-06 + | > step_time: 0.355 (0.31910311077916453) + | > loader_time: 0.0097 (0.009621164410613278) + + + --> TIME: 2024-10-20 18:45:45 -- STEP: 8650/16928 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.03255392611026764 (0.03560457760565005) + | > loss_mel_ce: 4.520595073699951 (4.236950678384385) + | > loss: 1.1382873058319092 (1.06813881421365) + | > current_lr: 5e-06 + | > step_time: 0.3442 (0.3190187807303643) + | > loader_time: 0.0087 (0.00961757466972216) + + + --> TIME: 2024-10-20 18:46:12 -- STEP: 8700/16928 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.03435952588915825 (0.03560492299861071) + | > loss_mel_ce: 3.6752402782440186 (4.2359274019866255) + | > loss: 0.9273999333381653 (1.0678830814841174) + | > current_lr: 5e-06 + | > step_time: 0.2317 (0.31900960037078074) + | > loader_time: 0.0093 (0.009612845963445173) + + + --> TIME: 2024-10-20 18:46:37 -- STEP: 8750/16928 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.03296118602156639 (0.03560350161045768) + | > loss_mel_ce: 3.98942232131958 (4.235113380186909) + | > loss: 1.0055959224700928 (1.0676792207241073) + | > current_lr: 5e-06 + | > step_time: 0.1715 (0.3189315682547442) + | > loader_time: 0.0078 (0.009607784925188304) + + + --> TIME: 2024-10-20 18:47:04 -- STEP: 8800/16928 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.03520086780190468 (0.03560185406653359) + | > loss_mel_ce: 4.744638442993164 (4.234342261038053) + | > loss: 1.1949598789215088 (1.0674860290844355) + | > current_lr: 5e-06 + | > step_time: 0.289 (0.31888986441222217) + | > loader_time: 0.0089 (0.009603719196536293) + + + --> TIME: 2024-10-20 18:47:30 -- STEP: 8850/16928 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.032328903675079346 (0.03559832998419864) + | > loss_mel_ce: 3.88148832321167 (4.233366847577076) + | > loss: 0.9784542918205261 (1.0672412947140195) + | > current_lr: 5e-06 + | > step_time: 0.2292 (0.3188389054530096) + | > loader_time: 0.0082 (0.009600842147223657) + + + --> TIME: 2024-10-20 18:47:56 -- STEP: 8900/16928 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.035821445286273956 (0.03559812517732024) + | > loss_mel_ce: 4.432693958282471 (4.232501121665663) + | > loss: 1.117128849029541 (1.0670248120286512) + | > current_lr: 5e-06 + | > step_time: 0.3068 (0.318813406483512) + | > loader_time: 0.0097 (0.00959800621097004) + + + --> TIME: 2024-10-20 18:48:24 -- STEP: 8950/16928 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.03472456336021423 (0.03559570106297877) + | > loss_mel_ce: 3.8637635707855225 (4.231508770868108) + | > loss: 0.9746220111846924 (1.0667761182785058) + | > current_lr: 5e-06 + | > step_time: 0.5233 (0.3188716615378537) + | > loader_time: 0.0102 (0.009595274685481369) + + + --> TIME: 2024-10-20 18:48:49 -- STEP: 9000/16928 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.03329377993941307 (0.03559527225337073) + | > loss_mel_ce: 4.122742176055908 (4.230758312410789) + | > loss: 1.0390089750289917 (1.0665883964631317) + | > current_lr: 5e-06 + | > step_time: 0.168 (0.31877248459392254) + | > loader_time: 0.012 (0.009592037783728663) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_9000.pth + + --> TIME: 2024-10-20 18:49:27 -- STEP: 9050/16928 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.04373094066977501 (0.035592460242094684) + | > loss_mel_ce: 3.970644950866699 (4.229753265907762) + | > loss: 1.003593921661377 (1.0663364318326063) + | > current_lr: 5e-06 + | > step_time: 0.3132 (0.3187666826617006) + | > loader_time: 0.0088 (0.00959110491842192) + + + --> TIME: 2024-10-20 18:49:53 -- STEP: 9100/16928 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.03767789527773857 (0.03559314798911661) + | > loss_mel_ce: 4.669374942779541 (4.22864507533693) + | > loss: 1.1767631769180298 (1.0660595561187372) + | > current_lr: 5e-06 + | > step_time: 0.1997 (0.3187136211762074) + | > loader_time: 0.0088 (0.009588104316166425) + + + --> TIME: 2024-10-20 18:50:21 -- STEP: 9150/16928 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.037246882915496826 (0.03559378617285354) + | > loss_mel_ce: 3.960158348083496 (4.227629701166218) + | > loss: 0.9993513226509094 (1.0658058720989978) + | > current_lr: 5e-06 + | > step_time: 0.4415 (0.3188028955459608) + | > loader_time: 0.0095 (0.009586074651916125) + + + --> TIME: 2024-10-20 18:50:51 -- STEP: 9200/16928 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.03900211676955223 (0.03559281526659792) + | > loss_mel_ce: 3.9831326007843018 (4.22659137360429) + | > loss: 1.0055336952209473 (1.0655460475126057) + | > current_lr: 5e-06 + | > step_time: 0.221 (0.31900769308857413) + | > loader_time: 0.0086 (0.009584332315818086) + + + --> TIME: 2024-10-20 18:51:18 -- STEP: 9250/16928 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.03324190154671669 (0.03559250686740529) + | > loss_mel_ce: 3.7780988216400146 (4.225506789697192) + | > loss: 0.952835202217102 (1.0652748244582015) + | > current_lr: 5e-06 + | > step_time: 0.3147 (0.31903379378447777) + | > loader_time: 0.0085 (0.009582362664712407) + + + --> TIME: 2024-10-20 18:51:45 -- STEP: 9300/16928 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.036961350589990616 (0.035589698164733584) + | > loss_mel_ce: 4.236330986022949 (4.224552839494529) + | > loss: 1.0683231353759766 (1.0650356347522447) + | > current_lr: 5e-06 + | > step_time: 0.1848 (0.31909552263957314) + | > loader_time: 0.0085 (0.009580248889102692) + + + --> TIME: 2024-10-20 18:52:12 -- STEP: 9350/16928 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.03352874144911766 (0.03558523181048922) + | > loss_mel_ce: 3.675814628601074 (4.2237225197215835) + | > loss: 0.9273358583450317 (1.0648269382135134) + | > current_lr: 5e-06 + | > step_time: 0.3004 (0.31910325124302147) + | > loader_time: 0.0094 (0.009578391421924896) + + + --> TIME: 2024-10-20 18:52:40 -- STEP: 9400/16928 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.032822299748659134 (0.035579854208341734) + | > loss_mel_ce: 3.922978401184082 (4.222489393974878) + | > loss: 0.9889501929283142 (1.0645173123605725) + | > current_lr: 5e-06 + | > step_time: 0.4165 (0.3191596819238474) + | > loader_time: 0.0098 (0.009576415026441496) + + + --> TIME: 2024-10-20 18:53:07 -- STEP: 9450/16928 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.03285524249076843 (0.03557831371035507) + | > loss_mel_ce: 3.6822924613952637 (4.221630513579761) + | > loss: 0.9287869334220886 (1.0643022070801453) + | > current_lr: 5e-06 + | > step_time: 0.2159 (0.3191805691946135) + | > loader_time: 0.0083 (0.009573684369445442) + + + --> TIME: 2024-10-20 18:53:35 -- STEP: 9500/16928 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.040030427277088165 (0.03557627829223069) + | > loss_mel_ce: 4.73688268661499 (4.2207580598028125) + | > loss: 1.1942282915115356 (1.0640835847854628) + | > current_lr: 5e-06 + | > step_time: 0.2496 (0.3193161862022013) + | > loader_time: 0.0091 (0.009571159463179704) + + + --> TIME: 2024-10-20 18:54:03 -- STEP: 9550/16928 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.03388757258653641 (0.035573980980869474) + | > loss_mel_ce: 4.577986240386963 (4.220045977512585) + | > loss: 1.152968406677246 (1.0639049898829152) + | > current_lr: 5e-06 + | > step_time: 0.2328 (0.31934112476428916) + | > loader_time: 0.0084 (0.009568997627777543) + + + --> TIME: 2024-10-20 18:54:29 -- STEP: 9600/16928 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.03820113092660904 (0.03557264778646622) + | > loss_mel_ce: 3.699552536010742 (4.218929657066866) + | > loss: 0.9344384074211121 (1.0636255765023346) + | > current_lr: 5e-06 + | > step_time: 0.1531 (0.3193432923903083) + | > loader_time: 0.01 (0.00956740610301491) + + + --> TIME: 2024-10-20 18:54:57 -- STEP: 9650/16928 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.03380436450242996 (0.03556863652185129) + | > loss_mel_ce: 3.839463710784912 (4.217998988245438) + | > loss: 0.9683170318603516 (1.0633919064677446) + | > current_lr: 5e-06 + | > step_time: 0.3092 (0.3193914750697098) + | > loader_time: 0.0086 (0.009564658199567215) + + + --> TIME: 2024-10-20 18:55:26 -- STEP: 9700/16928 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.03439510241150856 (0.035569214851201625) + | > loss_mel_ce: 3.871772527694702 (4.217108796871818) + | > loss: 0.9765419363975525 (1.0631695032058313) + | > current_lr: 5e-06 + | > step_time: 0.2079 (0.3195444815920816) + | > loader_time: 0.0092 (0.009562490551742048) + + + --> TIME: 2024-10-20 18:55:53 -- STEP: 9750/16928 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.03615174815058708 (0.03556852545379043) + | > loss_mel_ce: 4.172138214111328 (4.216040088629116) + | > loss: 1.052072525024414 (1.0629021538037526) + | > current_lr: 5e-06 + | > step_time: 0.2898 (0.3195852834505931) + | > loader_time: 0.0087 (0.009560083511548126) + + + --> TIME: 2024-10-20 18:56:20 -- STEP: 9800/16928 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.03577472269535065 (0.03556574864084904) + | > loss_mel_ce: 3.988095760345459 (4.2152067107327165) + | > loss: 1.005967617034912 (1.0626931151321959) + | > current_lr: 5e-06 + | > step_time: 0.3982 (0.3195924952808708) + | > loader_time: 0.0094 (0.009558036351690452) + + + --> TIME: 2024-10-20 18:56:46 -- STEP: 9850/16928 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.036345213651657104 (0.03556593614200173) + | > loss_mel_ce: 4.110727310180664 (4.214200528648305) + | > loss: 1.0367680788040161 (1.062441616500091) + | > current_lr: 5e-06 + | > step_time: 0.3419 (0.3195397045890708) + | > loader_time: 0.0093 (0.009556078039450052) + + + --> TIME: 2024-10-20 18:57:13 -- STEP: 9900/16928 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.03496989235281944 (0.03556649440899468) + | > loss_mel_ce: 4.67538595199585 (4.213048499064014) + | > loss: 1.177588939666748 (1.0621537486772357) + | > current_lr: 5e-06 + | > step_time: 0.2025 (0.31954097425094913) + | > loader_time: 0.0085 (0.009553368284244693) + + + --> TIME: 2024-10-20 18:57:38 -- STEP: 9950/16928 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.036461036652326584 (0.035565448386322966) + | > loss_mel_ce: 3.9740641117095947 (4.212099763975671) + | > loss: 1.0026313066482544 (1.0619163034189896) + | > current_lr: 5e-06 + | > step_time: 0.2161 (0.31939840141852444) + | > loader_time: 0.0089 (0.009549934636408316) + + + --> TIME: 2024-10-20 18:58:04 -- STEP: 10000/16928 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.03276987373828888 (0.03556429841872283) + | > loss_mel_ce: 3.7381608486175537 (4.210973610115052) + | > loss: 0.9427326917648315 (1.0616344774663449) + | > current_lr: 5e-06 + | > step_time: 0.1704 (0.31936333036422865) + | > loader_time: 0.0092 (0.009547849464416468) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_10000.pth + + --> TIME: 2024-10-20 18:58:45 -- STEP: 10050/16928 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.03463936597108841 (0.03556348084439665) + | > loss_mel_ce: 3.943176746368408 (4.210142010005552) + | > loss: 0.994454026222229 (1.0614263730262645) + | > current_lr: 5e-06 + | > step_time: 0.2618 (0.3193785608585806) + | > loader_time: 0.0085 (0.009548259872702185) + + + --> TIME: 2024-10-20 18:59:13 -- STEP: 10100/16928 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.039164893329143524 (0.03555844009660238) + | > loss_mel_ce: 3.9515936374664307 (4.208824018653076) + | > loss: 0.9976896047592163 (1.061095614993927) + | > current_lr: 5e-06 + | > step_time: 0.3051 (0.3194479418981204) + | > loader_time: 0.01 (0.009546267443364169) + + + --> TIME: 2024-10-20 18:59:39 -- STEP: 10150/16928 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.02893216721713543 (0.035554952301048104) + | > loss_mel_ce: 4.790743350982666 (4.207896634111262) + | > loss: 1.2049188613891602 (1.0608628968887153) + | > current_lr: 5e-06 + | > step_time: 0.2654 (0.31943427642578104) + | > loader_time: 0.0085 (0.009543753116588834) + + + --> TIME: 2024-10-20 19:00:06 -- STEP: 10200/16928 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.03889884427189827 (0.0355518270316806) + | > loss_mel_ce: 4.915892601013184 (4.2074306415576554) + | > loss: 1.23869788646698 (1.0607456174083794) + | > current_lr: 5e-06 + | > step_time: 0.2543 (0.31944800157173125) + | > loader_time: 0.0084 (0.009541416846069604) + + + --> TIME: 2024-10-20 19:00:32 -- STEP: 10250/16928 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.031816914677619934 (0.035548030417684964) + | > loss_mel_ce: 4.03285026550293 (4.206290634969387) + | > loss: 1.0161668062210083 (1.0604596666010422) + | > current_lr: 5e-06 + | > step_time: 0.3619 (0.31934776980702584) + | > loader_time: 0.0093 (0.00953916982325108) + + + --> TIME: 2024-10-20 19:00:59 -- STEP: 10300/16928 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.03270942345261574 (0.035544580815775745) + | > loss_mel_ce: 3.7340285778045654 (4.20538014506831) + | > loss: 0.9416844844818115 (1.0602311817187708) + | > current_lr: 5e-06 + | > step_time: 0.2055 (0.3193597410720541) + | > loader_time: 0.0096 (0.009535409640339935) + + + --> TIME: 2024-10-20 19:01:26 -- STEP: 10350/16928 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.03579551354050636 (0.03554247570606517) + | > loss_mel_ce: 3.8897275924682617 (4.204296406662987) + | > loss: 0.9813807606697083 (1.0599597208338656) + | > current_lr: 5e-06 + | > step_time: 0.288 (0.3193934117883887) + | > loader_time: 0.0098 (0.009532607926262721) + + + --> TIME: 2024-10-20 19:01:53 -- STEP: 10400/16928 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.03155644237995148 (0.0355435788728152) + | > loss_mel_ce: 4.139793395996094 (4.203625494448045) + | > loss: 1.0428375005722046 (1.0597922685685062) + | > current_lr: 5e-06 + | > step_time: 0.3202 (0.319428465480989) + | > loader_time: 0.0095 (0.00952939634139718) + + + --> TIME: 2024-10-20 19:02:20 -- STEP: 10450/16928 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.03365842625498772 (0.035538120115594554) + | > loss_mel_ce: 4.128153324127197 (4.203020360139003) + | > loss: 1.0404529571533203 (1.0596396203132334) + | > current_lr: 5e-06 + | > step_time: 0.2995 (0.3194588616247961) + | > loader_time: 0.0089 (0.009526502458672741) + + + --> TIME: 2024-10-20 19:02:48 -- STEP: 10500/16928 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.03561829403042793 (0.035533430408686256) + | > loss_mel_ce: 4.753257751464844 (4.2020742823509964) + | > loss: 1.1972190141677856 (1.0594019284418643) + | > current_lr: 5e-06 + | > step_time: 0.2089 (0.3194919714700618) + | > loader_time: 0.0087 (0.009523751031784752) + + + --> TIME: 2024-10-20 19:03:14 -- STEP: 10550/16928 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.0387689583003521 (0.0355282186188897) + | > loss_mel_ce: 3.850132703781128 (4.20117646597008) + | > loss: 0.9722254276275635 (1.059176171362682) + | > current_lr: 5e-06 + | > step_time: 0.2325 (0.31947105706020645) + | > loader_time: 0.0087 (0.009521095673619829) + + + --> TIME: 2024-10-20 19:03:42 -- STEP: 10600/16928 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.03519758582115173 (0.035526629478828654) + | > loss_mel_ce: 4.291449546813965 (4.200598635763493) + | > loss: 1.0816618204116821 (1.0590313165255305) + | > current_lr: 5e-06 + | > step_time: 0.287 (0.31957413968050363) + | > loader_time: 0.0087 (0.009518918541242458) + + + --> TIME: 2024-10-20 19:04:10 -- STEP: 10650/16928 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.033840835094451904 (0.03552658922917813) + | > loss_mel_ce: 3.2568185329437256 (4.199463381229991) + | > loss: 0.8226648569107056 (1.0587474928159664) + | > current_lr: 5e-06 + | > step_time: 0.248 (0.3196068488152386) + | > loader_time: 0.0087 (0.009515629240044935) + + + --> TIME: 2024-10-20 19:04:36 -- STEP: 10700/16928 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.03682931140065193 (0.03552884957552917) + | > loss_mel_ce: 3.679286479949951 (4.19854495870733) + | > loss: 0.9290289282798767 (1.0585184522766922) + | > current_lr: 5e-06 + | > step_time: 0.3494 (0.3195858959839736) + | > loader_time: 0.0108 (0.009514005674379974) + + + --> TIME: 2024-10-20 19:05:03 -- STEP: 10750/16928 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.03489922359585762 (0.035526140022069747) + | > loss_mel_ce: 4.555095195770264 (4.197924889830656) + | > loss: 1.147498607635498 (1.0583627576938883) + | > current_lr: 5e-06 + | > step_time: 0.5125 (0.3196080940157881) + | > loader_time: 0.0106 (0.00951202982525489) + + + --> TIME: 2024-10-20 19:05:31 -- STEP: 10800/16928 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.036657270044088364 (0.03552391176957071) + | > loss_mel_ce: 4.095895290374756 (4.19710538307826) + | > loss: 1.0331381559371948 (1.0581573239686295) + | > current_lr: 5e-06 + | > step_time: 0.322 (0.3196329496304207) + | > loader_time: 0.0093 (0.009510257994687077) + + + --> TIME: 2024-10-20 19:05:57 -- STEP: 10850/16928 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.04175588861107826 (0.03552074415137128) + | > loss_mel_ce: 4.038936614990234 (4.196419658353253) + | > loss: 1.0201730728149414 (1.0579851008670105) + | > current_lr: 5e-06 + | > step_time: 0.2303 (0.3196188965590882) + | > loader_time: 0.0084 (0.00950782211145494) + + + --> TIME: 2024-10-20 19:06:23 -- STEP: 10900/16928 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.03273124247789383 (0.03551846495803878) + | > loss_mel_ce: 4.323493003845215 (4.195909830955192) + | > loss: 1.0890560150146484 (1.0578570741743112) + | > current_lr: 5e-06 + | > step_time: 0.247 (0.3195937376284832) + | > loader_time: 0.0086 (0.009505683063367056) + + + --> TIME: 2024-10-20 19:06:50 -- STEP: 10950/16928 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.033156514167785645 (0.03551837648379871) + | > loss_mel_ce: 3.823544979095459 (4.1948316948163455) + | > loss: 0.9641753435134888 (1.057587517989824) + | > current_lr: 5e-06 + | > step_time: 0.4284 (0.319630446542889) + | > loader_time: 0.0089 (0.009503959063525576) + + + --> TIME: 2024-10-20 19:07:17 -- STEP: 11000/16928 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.03703553229570389 (0.03551605858907767) + | > loss_mel_ce: 3.8676209449768066 (4.193991867325525) + | > loss: 0.9761641025543213 (1.0573769816051826) + | > current_lr: 5e-06 + | > step_time: 0.3047 (0.3196321801705803) + | > loader_time: 0.0094 (0.009502330563285097) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_11000.pth + + --> TIME: 2024-10-20 19:07:57 -- STEP: 11050/16928 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.03126528859138489 (0.03551535903244389) + | > loss_mel_ce: 4.139996528625488 (4.192975379611578) + | > loss: 1.0428154468536377 (1.0571226847603301) + | > current_lr: 5e-06 + | > step_time: 0.379 (0.3195415836437803) + | > loader_time: 0.0087 (0.009501519893628942) + + + --> TIME: 2024-10-20 19:08:23 -- STEP: 11100/16928 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.03693367540836334 (0.035511734120648335) + | > loss_mel_ce: 3.8977394104003906 (4.191953994948591) + | > loss: 0.9836682677268982 (1.0568664323854007) + | > current_lr: 5e-06 + | > step_time: 0.2034 (0.3195190051224866) + | > loader_time: 0.0084 (0.009498607510918942) + + + --> TIME: 2024-10-20 19:08:50 -- STEP: 11150/16928 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.0366506390273571 (0.0355098688919848) + | > loss_mel_ce: 3.649303674697876 (4.191311413739303) + | > loss: 0.9214885830879211 (1.0567053207741721) + | > current_lr: 5e-06 + | > step_time: 0.2536 (0.31951359524320716) + | > loader_time: 0.0085 (0.00949556716354436) + + + --> TIME: 2024-10-20 19:09:16 -- STEP: 11200/16928 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.035503558814525604 (0.03550803600328169) + | > loss_mel_ce: 4.0130205154418945 (4.19077216393182) + | > loss: 1.0121309757232666 (1.056570050099067) + | > current_lr: 5e-06 + | > step_time: 0.2305 (0.319471814802716) + | > loader_time: 0.0086 (0.009492299492870027) + + + --> TIME: 2024-10-20 19:09:41 -- STEP: 11250/16928 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.037533409893512726 (0.035506633441150105) + | > loss_mel_ce: 4.263673782348633 (4.1898767786449955) + | > loss: 1.0753017663955688 (1.056345853143268) + | > current_lr: 5e-06 + | > step_time: 0.3589 (0.3193880031373778) + | > loader_time: 0.0082 (0.009488126754760714) + + + --> TIME: 2024-10-20 19:10:08 -- STEP: 11300/16928 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.03575047105550766 (0.03550636947962687) + | > loss_mel_ce: 4.233911991119385 (4.189164207973318) + | > loss: 1.0674155950546265 (1.0561676444838517) + | > current_lr: 5e-06 + | > step_time: 0.2005 (0.3193772997265383) + | > loader_time: 0.0083 (0.009485254540907574) + + + --> TIME: 2024-10-20 19:10:34 -- STEP: 11350/16928 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.03519954904913902 (0.03550486348306812) + | > loss_mel_ce: 3.671823024749756 (4.1882716865791645) + | > loss: 0.9267556667327881 (1.0559441376415095) + | > current_lr: 5e-06 + | > step_time: 0.2156 (0.3193889168390634) + | > loader_time: 0.0082 (0.00948268022831314) + + + --> TIME: 2024-10-20 19:11:01 -- STEP: 11400/16928 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.03477318212389946 (0.03550197775018184) + | > loss_mel_ce: 4.444644927978516 (4.18747613814839) + | > loss: 1.1198545694351196 (1.0557445290841534) + | > current_lr: 5e-06 + | > step_time: 0.3019 (0.31937291655624295) + | > loader_time: 0.0082 (0.00947989854896275) + + + --> TIME: 2024-10-20 19:11:27 -- STEP: 11450/16928 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.032973967492580414 (0.035500075431727625) + | > loss_mel_ce: 4.010453224182129 (4.186529562421238) + | > loss: 1.0108567476272583 (1.0555074095621872) + | > current_lr: 5e-06 + | > step_time: 0.2307 (0.31932497228597634) + | > loader_time: 0.0085 (0.009476538120919415) + + + --> TIME: 2024-10-20 19:11:51 -- STEP: 11500/16928 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.033651527017354965 (0.03549676483432232) + | > loss_mel_ce: 4.576859951019287 (4.185639083053748) + | > loss: 1.1526278257369995 (1.0552839620631649) + | > current_lr: 5e-06 + | > step_time: 0.2442 (0.31921188153391467) + | > loader_time: 0.0086 (0.00947304617840308) + + + --> TIME: 2024-10-20 19:12:19 -- STEP: 11550/16928 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.032848700881004333 (0.03549450360512702) + | > loss_mel_ce: 3.68741512298584 (4.184801413858082) + | > loss: 0.930065929889679 (1.0550739794427677) + | > current_lr: 5e-06 + | > step_time: 0.3567 (0.31925704477153477) + | > loader_time: 0.0093 (0.009471300092094365) + + + --> TIME: 2024-10-20 19:12:46 -- STEP: 11600/16928 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.04234845191240311 (0.035494775432483076) + | > loss_mel_ce: 3.6372764110565186 (4.183629255335899) + | > loss: 0.9199061989784241 (1.0547810077564461) + | > current_lr: 5e-06 + | > step_time: 0.2226 (0.3192854617381934) + | > loader_time: 0.0089 (0.00946902287417442) + + + --> TIME: 2024-10-20 19:13:12 -- STEP: 11650/16928 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.031014373525977135 (0.03549155169946055) + | > loss_mel_ce: 3.710911512374878 (4.18302916209584) + | > loss: 0.9354814887046814 (1.0546301785559107) + | > current_lr: 5e-06 + | > step_time: 0.3392 (0.31925926574821806) + | > loader_time: 0.0089 (0.009466063270241371) + + + --> TIME: 2024-10-20 19:13:39 -- STEP: 11700/16928 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.03830355033278465 (0.035491317685566105) + | > loss_mel_ce: 3.9560117721557617 (4.181907963569339) + | > loss: 0.9985788464546204 (1.0543498203948012) + | > current_lr: 5e-06 + | > step_time: 0.3005 (0.31929067236745634) + | > loader_time: 0.0085 (0.009462411566677213) + + + --> TIME: 2024-10-20 19:14:07 -- STEP: 11750/16928 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.03268520534038544 (0.035488199903768385) + | > loss_mel_ce: 3.9575815200805664 (4.181106991849037) + | > loss: 0.9975666999816895 (1.0541487980142554) + | > current_lr: 5e-06 + | > step_time: 0.4444 (0.31935845872189295) + | > loader_time: 0.0106 (0.009459351194665741) + + + --> TIME: 2024-10-20 19:14:34 -- STEP: 11800/16928 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.03590023145079613 (0.03548543609947085) + | > loss_mel_ce: 4.073548316955566 (4.1804216699883) + | > loss: 1.0273621082305908 (1.0539767765847314) + | > current_lr: 5e-06 + | > step_time: 0.1841 (0.31937706171456176) + | > loader_time: 0.0085 (0.00945709616450936) + + + --> TIME: 2024-10-20 19:15:01 -- STEP: 11850/16928 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.029822299256920815 (0.035483640930477754) + | > loss_mel_ce: 4.253194808959961 (4.179646635055531) + | > loss: 1.0707542896270752 (1.0537825690543106) + | > current_lr: 5e-06 + | > step_time: 0.3384 (0.31936126507787405) + | > loader_time: 0.0082 (0.009453623304890135) + + + --> TIME: 2024-10-20 19:15:29 -- STEP: 11900/16928 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.03357449918985367 (0.035479631428802394) + | > loss_mel_ce: 3.955491781234741 (4.1791272603363465) + | > loss: 0.9972665905952454 (1.053651723000182) + | > current_lr: 5e-06 + | > step_time: 0.3453 (0.3194246267871712) + | > loader_time: 0.0088 (0.009450701425055453) + + + --> TIME: 2024-10-20 19:15:56 -- STEP: 11950/16928 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.02945839986205101 (0.0354775558977882) + | > loss_mel_ce: 3.8668627738952637 (4.178345924401374) + | > loss: 0.9740802645683289 (1.0534558701515193) + | > current_lr: 5e-06 + | > step_time: 0.4301 (0.31946183529857963) + | > loader_time: 0.0091 (0.009447975338253471) + + + --> TIME: 2024-10-20 19:16:21 -- STEP: 12000/16928 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.035935256630182266 (0.035477051596778074) + | > loss_mel_ce: 4.262418270111084 (4.177302540977789) + | > loss: 1.074588418006897 (1.0531948982079817) + | > current_lr: 5e-06 + | > step_time: 0.2048 (0.3193333438237527) + | > loader_time: 0.0091 (0.009443960924943245) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_12000.pth + + --> TIME: 2024-10-20 19:17:01 -- STEP: 12050/16928 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.03786998987197876 (0.035474979849338964) + | > loss_mel_ce: 4.360647678375244 (4.176838052045254) + | > loss: 1.0996294021606445 (1.0530782580326201) + | > current_lr: 5e-06 + | > step_time: 0.5031 (0.3193146456326706) + | > loader_time: 0.009 (0.009444037196052484) + + + --> TIME: 2024-10-20 19:17:29 -- STEP: 12100/16928 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.033528055995702744 (0.035472027520564414) + | > loss_mel_ce: 4.290019512176514 (4.176383486720147) + | > loss: 1.0808868408203125 (1.0529638786109023) + | > current_lr: 5e-06 + | > step_time: 0.3963 (0.3193376956498348) + | > loader_time: 0.0095 (0.009441842481124469) + + + --> TIME: 2024-10-20 19:17:57 -- STEP: 12150/16928 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.03534900024533272 (0.035471080564452986) + | > loss_mel_ce: 3.722360134124756 (4.17564794036095) + | > loss: 0.9394272565841675 (1.0527797552734728) + | > current_lr: 5e-06 + | > step_time: 0.3029 (0.31940000557605347) + | > loader_time: 0.0086 (0.009439574190618534) + + + --> TIME: 2024-10-20 19:18:24 -- STEP: 12200/16928 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.03367210179567337 (0.035468139082926196) + | > loss_mel_ce: 3.691136121749878 (4.17496361017226) + | > loss: 0.9312020540237427 (1.0526079373476922) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.319465849946759) + | > loader_time: 0.0084 (0.00943627044802803) + + + --> TIME: 2024-10-20 19:18:50 -- STEP: 12250/16928 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.0313449501991272 (0.03546602624274635) + | > loss_mel_ce: 4.161384582519531 (4.174334317499264) + | > loss: 1.0481823682785034 (1.052450085985415) + | > current_lr: 5e-06 + | > step_time: 0.2294 (0.319376441702553) + | > loader_time: 0.0093 (0.009435249425926955) + + + --> TIME: 2024-10-20 19:19:17 -- STEP: 12300/16928 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.03531685099005699 (0.03546485429523677) + | > loss_mel_ce: 3.839138984680176 (4.173557041951297) + | > loss: 0.9686139822006226 (1.05225547410608) + | > current_lr: 5e-06 + | > step_time: 0.2718 (0.3194007659539967) + | > loader_time: 0.0081 (0.00943451706956069) + + + --> TIME: 2024-10-20 19:19:43 -- STEP: 12350/16928 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.032652415335178375 (0.03546375320311864) + | > loss_mel_ce: 4.219954013824463 (4.172754294283464) + | > loss: 1.0631515979766846 (1.0520545119360836) + | > current_lr: 5e-06 + | > step_time: 0.215 (0.31937948510714653) + | > loader_time: 0.0088 (0.00943345116217606) + + + --> TIME: 2024-10-20 19:20:09 -- STEP: 12400/16928 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.03542506322264671 (0.035461420696228776) + | > loss_mel_ce: 3.830430030822754 (4.171849536703464) + | > loss: 0.9664637446403503 (1.051827739424279) + | > current_lr: 5e-06 + | > step_time: 0.2047 (0.31931803201475684) + | > loader_time: 0.0091 (0.009431526372509576) + + + --> TIME: 2024-10-20 19:20:36 -- STEP: 12450/16928 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.035703569650650024 (0.035460147520295374) + | > loss_mel_ce: 4.139565944671631 (4.171085170799458) + | > loss: 1.043817400932312 (1.051636329654706) + | > current_lr: 5e-06 + | > step_time: 0.337 (0.3193381888416424) + | > loader_time: 0.0087 (0.009430749406776233) + + + --> TIME: 2024-10-20 19:21:03 -- STEP: 12500/16928 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.03336132690310478 (0.035455200212448856) + | > loss_mel_ce: 3.9137589931488037 (4.170267991085047) + | > loss: 0.9867801070213318 (1.051430797901151) + | > current_lr: 5e-06 + | > step_time: 0.2439 (0.31935784584045673) + | > loader_time: 0.0078 (0.009427793254852249) + + + --> TIME: 2024-10-20 19:21:29 -- STEP: 12550/16928 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.03596093878149986 (0.035453193224224466) + | > loss_mel_ce: 4.008167266845703 (4.169649678872392) + | > loss: 1.0110321044921875 (1.0512757181028904) + | > current_lr: 5e-06 + | > step_time: 0.2495 (0.3193047185153153) + | > loader_time: 0.0091 (0.009425120239713711) + + + --> TIME: 2024-10-20 19:21:54 -- STEP: 12600/16928 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.03832113370299339 (0.03545173165329273) + | > loss_mel_ce: 4.062595844268799 (4.1687107280511615) + | > loss: 1.0252292156219482 (1.0510406149805527) + | > current_lr: 5e-06 + | > step_time: 0.2219 (0.31923610312598344) + | > loader_time: 0.0084 (0.009422013135183389) + + + --> TIME: 2024-10-20 19:22:21 -- STEP: 12650/16928 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.035744886845350266 (0.03544953961470145) + | > loss_mel_ce: 3.675941228866577 (4.167805878635444) + | > loss: 0.9279215335845947 (1.0508138546594978) + | > current_lr: 5e-06 + | > step_time: 0.3004 (0.3192612498739517) + | > loader_time: 0.0079 (0.00941962522951507) + + + --> TIME: 2024-10-20 19:22:48 -- STEP: 12700/16928 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.035050224512815475 (0.035446618342346775) + | > loss_mel_ce: 3.8832762241363525 (4.166879709179934) + | > loss: 0.9795815944671631 (1.0505815819867952) + | > current_lr: 5e-06 + | > step_time: 0.2501 (0.3192477896833069) + | > loader_time: 0.0083 (0.009416787436627931) + + + --> TIME: 2024-10-20 19:23:14 -- STEP: 12750/16928 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.03766316920518875 (0.035444508649116614) + | > loss_mel_ce: 3.6759603023529053 (4.165998269081106) + | > loss: 0.9284058809280396 (1.0503606945439878) + | > current_lr: 5e-06 + | > step_time: 0.2871 (0.3192530596490025) + | > loader_time: 0.0082 (0.009413836890575896) + + + --> TIME: 2024-10-20 19:23:41 -- STEP: 12800/16928 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.03564820811152458 (0.03544302081165368) + | > loss_mel_ce: 4.305308818817139 (4.165087722782036) + | > loss: 1.085239291191101 (1.0501326860161497) + | > current_lr: 5e-06 + | > step_time: 0.2884 (0.31924651427194733) + | > loader_time: 0.0083 (0.009411635715514382) + + + --> TIME: 2024-10-20 19:24:08 -- STEP: 12850/16928 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.03536717966198921 (0.03543845880834623) + | > loss_mel_ce: 4.223402500152588 (4.164437846784911) + | > loss: 1.0646923780441284 (1.049969076502648) + | > current_lr: 5e-06 + | > step_time: 0.3013 (0.319287352766047) + | > loader_time: 0.0082 (0.009408863546319464) + + + --> TIME: 2024-10-20 19:24:35 -- STEP: 12900/16928 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.03416738286614418 (0.0354370371826975) + | > loss_mel_ce: 4.232370376586914 (4.163939627381243) + | > loss: 1.0666344165802002 (1.0498441662668259) + | > current_lr: 5e-06 + | > step_time: 0.3754 (0.3192813136226472) + | > loader_time: 0.0086 (0.009406503060067309) + + + --> TIME: 2024-10-20 19:25:00 -- STEP: 12950/16928 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.034840699285268784 (0.03543471925564716) + | > loss_mel_ce: 4.242241382598877 (4.162934304296264) + | > loss: 1.0692704916000366 (1.0495922560167124) + | > current_lr: 5e-06 + | > step_time: 0.3784 (0.319186696766889) + | > loader_time: 0.0088 (0.009403976823372236) + + + --> TIME: 2024-10-20 19:25:27 -- STEP: 13000/16928 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.035327594727277756 (0.035430555768740886) + | > loss_mel_ce: 4.254801273345947 (4.162244779715151) + | > loss: 1.0725321769714355 (1.0494188340077033) + | > current_lr: 5e-06 + | > step_time: 0.1999 (0.3191787699552708) + | > loader_time: 0.0084 (0.00940119587458094) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_13000.pth + + --> TIME: 2024-10-20 19:26:08 -- STEP: 13050/16928 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.035458263009786606 (0.03542760354991278) + | > loss_mel_ce: 3.94939923286438 (4.161462777053692) + | > loss: 0.9962143898010254 (1.0492225952897503) + | > current_lr: 5e-06 + | > step_time: 0.3795 (0.3191727110435248) + | > loader_time: 0.0086 (0.009400876618893179) + + + --> TIME: 2024-10-20 19:26:33 -- STEP: 13100/16928 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.032693181186914444 (0.035424787879758037) + | > loss_mel_ce: 4.3292365074157715 (4.160450616119449) + | > loss: 1.090482473373413 (1.0489688511568185) + | > current_lr: 5e-06 + | > step_time: 0.2742 (0.319086759436224) + | > loader_time: 0.0088 (0.009397872695486026) + + + --> TIME: 2024-10-20 19:27:00 -- STEP: 13150/16928 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.036297012120485306 (0.03542392656717017) + | > loss_mel_ce: 4.170567035675049 (4.159850766921655) + | > loss: 1.0517159700393677 (1.0488186735345395) + | > current_lr: 5e-06 + | > step_time: 0.4415 (0.3190967056506512) + | > loader_time: 0.0091 (0.009395041864634447) + + + --> TIME: 2024-10-20 19:27:27 -- STEP: 13200/16928 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.035902224481105804 (0.035421939519883205) + | > loss_mel_ce: 3.744795799255371 (4.159205341212652) + | > loss: 0.945174515247345 (1.0486568203478166) + | > current_lr: 5e-06 + | > step_time: 0.2471 (0.31909471096414754) + | > loader_time: 0.0084 (0.009392582575480115) + + + --> TIME: 2024-10-20 19:27:54 -- STEP: 13250/16928 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.03486135974526405 (0.0354214104605172) + | > loss_mel_ce: 4.039559841156006 (4.158770531366429) + | > loss: 1.018605351448059 (1.048547985616721) + | > current_lr: 5e-06 + | > step_time: 0.228 (0.3191111925053171) + | > loader_time: 0.009 (0.009390214254271275) + + + --> TIME: 2024-10-20 19:28:20 -- STEP: 13300/16928 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.03699821978807449 (0.035420622924917024) + | > loss_mel_ce: 3.4199366569519043 (4.1579382757315635) + | > loss: 0.8642337322235107 (1.0483397248096051) + | > current_lr: 5e-06 + | > step_time: 0.2104 (0.3191016960681854) + | > loader_time: 0.0081 (0.009387860280230491) + + + --> TIME: 2024-10-20 19:28:48 -- STEP: 13350/16928 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.037235505878925323 (0.03541604314445894) + | > loss_mel_ce: 4.190235614776611 (4.15757025041827) + | > loss: 1.0568678379058838 (1.048246573553552) + | > current_lr: 5e-06 + | > step_time: 0.3124 (0.31914006286792784) + | > loader_time: 0.0079 (0.009385340508450248) + + + --> TIME: 2024-10-20 19:29:13 -- STEP: 13400/16928 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.033944014459848404 (0.0354127296154846) + | > loss_mel_ce: 3.9174678325653076 (4.157063204815108) + | > loss: 0.9878529906272888 (1.0481189837651486) + | > current_lr: 5e-06 + | > step_time: 0.2 (0.31907360260166584) + | > loader_time: 0.0077 (0.009382269062213002) + + + --> TIME: 2024-10-20 19:29:42 -- STEP: 13450/16928 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.03246290981769562 (0.035410025212983) + | > loss_mel_ce: 3.9786314964294434 (4.156438447409816) + | > loss: 1.002773642539978 (1.047962118311888) + | > current_lr: 5e-06 + | > step_time: 0.3948 (0.31916510018274485) + | > loader_time: 0.0091 (0.009380176076215392) + + + --> TIME: 2024-10-20 19:30:09 -- STEP: 13500/16928 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.03444599732756615 (0.03540896640055711) + | > loss_mel_ce: 3.618703603744507 (4.155595195558304) + | > loss: 0.9132874011993408 (1.0477510406661932) + | > current_lr: 5e-06 + | > step_time: 0.2866 (0.31918512130666904) + | > loader_time: 0.0084 (0.009377755783222287) + + + --> TIME: 2024-10-20 19:30:35 -- STEP: 13550/16928 -- GLOBAL_STEP: 13550 + | > loss_text_ce: 0.034270625561475754 (0.035406662821632226) + | > loss_mel_ce: 3.914583206176758 (4.154565101444026) + | > loss: 0.9872134327888489 (1.0474929412220695) + | > current_lr: 5e-06 + | > step_time: 0.3759 (0.3191535696508284) + | > loader_time: 0.0089 (0.009374434499283069) + + + --> TIME: 2024-10-20 19:31:01 -- STEP: 13600/16928 -- GLOBAL_STEP: 13600 + | > loss_text_ce: 0.03474517911672592 (0.035405486855272415) + | > loss_mel_ce: 3.755350112915039 (4.153842569116248) + | > loss: 0.947523832321167 (1.0473120141774435) + | > current_lr: 5e-06 + | > step_time: 0.208 (0.3191081923421715) + | > loader_time: 0.0082 (0.009371031873366365) + + + --> TIME: 2024-10-20 19:31:29 -- STEP: 13650/16928 -- GLOBAL_STEP: 13650 + | > loss_text_ce: 0.035578079521656036 (0.03540247890620659) + | > loss_mel_ce: 4.520397663116455 (4.153379373183581) + | > loss: 1.1389939785003662 (1.0471954631892777) + | > current_lr: 5e-06 + | > step_time: 0.4309 (0.3191920624023853) + | > loader_time: 0.009 (0.009368088778121952) + + + --> TIME: 2024-10-20 19:31:55 -- STEP: 13700/16928 -- GLOBAL_STEP: 13700 + | > loss_text_ce: 0.03914197161793709 (0.03540248925886013) + | > loss_mel_ce: 3.915308713912964 (4.152551566597282) + | > loss: 0.9886126518249512 (1.046988514126655) + | > current_lr: 5e-06 + | > step_time: 0.233 (0.3191573745664913) + | > loader_time: 0.0082 (0.009365513342140308) + + + --> TIME: 2024-10-20 19:32:22 -- STEP: 13750/16928 -- GLOBAL_STEP: 13750 + | > loss_text_ce: 0.03559420257806778 (0.03539968170306895) + | > loss_mel_ce: 4.446051120758057 (4.152003401478819) + | > loss: 1.1204112768173218 (1.0468507709719939) + | > current_lr: 5e-06 + | > step_time: 0.341 (0.3191619481520243) + | > loader_time: 0.0091 (0.009362952544472378) + + + --> TIME: 2024-10-20 19:32:49 -- STEP: 13800/16928 -- GLOBAL_STEP: 13800 + | > loss_text_ce: 0.03648313507437706 (0.03539822154624412) + | > loss_mel_ce: 3.685548782348633 (4.151321301892151) + | > loss: 0.9305079579353333 (1.0466798810207347) + | > current_lr: 5e-06 + | > step_time: 0.3959 (0.31917886447215665) + | > loader_time: 0.0092 (0.0093605960970339) + + + --> TIME: 2024-10-20 19:33:16 -- STEP: 13850/16928 -- GLOBAL_STEP: 13850 + | > loss_text_ce: 0.03573684021830559 (0.03539610843110275) + | > loss_mel_ce: 4.214633464813232 (4.150614017269639) + | > loss: 1.062592625617981 (1.0465025315818388) + | > current_lr: 5e-06 + | > step_time: 0.2331 (0.31918387042916874) + | > loader_time: 0.0077 (0.009358553542126762) + + + --> TIME: 2024-10-20 19:33:41 -- STEP: 13900/16928 -- GLOBAL_STEP: 13900 + | > loss_text_ce: 0.030053837224841118 (0.03539462177551905) + | > loss_mel_ce: 3.92154860496521 (4.150011111935235) + | > loss: 0.9879006147384644 (1.0463514335678652) + | > current_lr: 5e-06 + | > step_time: 0.183 (0.31910160416322025) + | > loader_time: 0.0085 (0.00935600431702972) + + + --> TIME: 2024-10-20 19:34:07 -- STEP: 13950/16928 -- GLOBAL_STEP: 13950 + | > loss_text_ce: 0.03833714500069618 (0.03539317283472281) + | > loss_mel_ce: 4.397568702697754 (4.149239006914087) + | > loss: 1.1089764833450317 (1.0461580450774117) + | > current_lr: 5e-06 + | > step_time: 0.3042 (0.3190875579977572) + | > loader_time: 0.0081 (0.009353328110069302) + + + --> TIME: 2024-10-20 19:34:35 -- STEP: 14000/16928 -- GLOBAL_STEP: 14000 + | > loss_text_ce: 0.03297251835465431 (0.03539099323989029) + | > loss_mel_ce: 3.9625744819641113 (4.148433875986477) + | > loss: 0.99888676404953 (1.0459562174252122) + | > current_lr: 5e-06 + | > step_time: 0.2285 (0.31911933457851643) + | > loader_time: 0.0098 (0.00935118334633958) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_14000.pth + + --> TIME: 2024-10-20 19:35:15 -- STEP: 14050/16928 -- GLOBAL_STEP: 14050 + | > loss_text_ce: 0.03615037724375725 (0.03538981832736225) + | > loss_mel_ce: 4.011239051818848 (4.1479427269599265) + | > loss: 1.0118473768234253 (1.0458331364414477) + | > current_lr: 5e-06 + | > step_time: 0.188 (0.3191149667828132) + | > loader_time: 0.0081 (0.00935083017654685) + + + --> TIME: 2024-10-20 19:35:43 -- STEP: 14100/16928 -- GLOBAL_STEP: 14100 + | > loss_text_ce: 0.035047031939029694 (0.03538735869215104) + | > loss_mel_ce: 3.6930181980133057 (4.147278983897317) + | > loss: 0.9320163130760193 (1.0456665857675254) + | > current_lr: 5e-06 + | > step_time: 0.2069 (0.31913408779929103) + | > loader_time: 0.0082 (0.009348497982566184) + + + --> TIME: 2024-10-20 19:36:08 -- STEP: 14150/16928 -- GLOBAL_STEP: 14150 + | > loss_text_ce: 0.037635743618011475 (0.03538610816054551) + | > loss_mel_ce: 4.266335964202881 (4.1467393239732) + | > loss: 1.0759929418563843 (1.0455313581796948) + | > current_lr: 5e-06 + | > step_time: 0.2317 (0.3190641556022047) + | > loader_time: 0.0082 (0.009345808753697641) + + + --> TIME: 2024-10-20 19:36:36 -- STEP: 14200/16928 -- GLOBAL_STEP: 14200 + | > loss_text_ce: 0.0368354506790638 (0.03538515785276158) + | > loss_mel_ce: 3.4536020755767822 (4.146052647426061) + | > loss: 0.8726093769073486 (1.0453594514685645) + | > current_lr: 5e-06 + | > step_time: 0.208 (0.3191241883224188) + | > loader_time: 0.0088 (0.009343642298604347) + + + --> TIME: 2024-10-20 19:37:00 -- STEP: 14250/16928 -- GLOBAL_STEP: 14250 + | > loss_text_ce: 0.03691699728369713 (0.03538448293141107) + | > loss_mel_ce: 3.988595485687256 (4.145248858317957) + | > loss: 1.006378173828125 (1.045158335459863) + | > current_lr: 5e-06 + | > step_time: 0.3581 (0.3190218659785779) + | > loader_time: 0.0085 (0.009341928047046272) + + + --> TIME: 2024-10-20 19:37:29 -- STEP: 14300/16928 -- GLOBAL_STEP: 14300 + | > loss_text_ce: 0.03271275386214256 (0.03538308778308801) + | > loss_mel_ce: 4.285861015319824 (4.144663983541735) + | > loss: 1.0796434879302979 (1.0450117679826056) + | > current_lr: 5e-06 + | > step_time: 0.2087 (0.31909489394901747) + | > loader_time: 0.0087 (0.00934045529865713) + + + --> TIME: 2024-10-20 19:37:54 -- STEP: 14350/16928 -- GLOBAL_STEP: 14350 + | > loss_text_ce: 0.0334489569067955 (0.035380069078806184) + | > loss_mel_ce: 4.068371295928955 (4.143897164922942) + | > loss: 1.025455117225647 (1.0448193086481283) + | > current_lr: 5e-06 + | > step_time: 0.3125 (0.31903377727349275) + | > loader_time: 0.0082 (0.009337747420998762) + + + --> TIME: 2024-10-20 19:38:21 -- STEP: 14400/16928 -- GLOBAL_STEP: 14400 + | > loss_text_ce: 0.03442253917455673 (0.035377802483142505) + | > loss_mel_ce: 3.6476125717163086 (4.143134908593332) + | > loss: 0.9205088019371033 (1.0446281779102171) + | > current_lr: 5e-06 + | > step_time: 0.3995 (0.3190480821000228) + | > loader_time: 0.0092 (0.009336246169275654) + + + --> TIME: 2024-10-20 19:38:47 -- STEP: 14450/16928 -- GLOBAL_STEP: 14450 + | > loss_text_ce: 0.03735670819878578 (0.035376796030719845) + | > loss_mel_ce: 3.481313467025757 (4.142376525393785) + | > loss: 0.8796675205230713 (1.04443833049606) + | > current_lr: 5e-06 + | > step_time: 0.1479 (0.31901721744801576) + | > loader_time: 0.0073 (0.009333536418663949) + + + --> TIME: 2024-10-20 19:39:16 -- STEP: 14500/16928 -- GLOBAL_STEP: 14500 + | > loss_text_ce: 0.03542214632034302 (0.035375279215003955) + | > loss_mel_ce: 4.046560764312744 (4.141775914521015) + | > loss: 1.0204957723617554 (1.0442877985444572) + | > current_lr: 5e-06 + | > step_time: 0.2747 (0.31910330857901964) + | > loader_time: 0.0085 (0.009331873219588654) + + + --> TIME: 2024-10-20 19:39:43 -- STEP: 14550/16928 -- GLOBAL_STEP: 14550 + | > loss_text_ce: 0.034205760806798935 (0.03537496177204393) + | > loss_mel_ce: 4.002822399139404 (4.141049927141211) + | > loss: 1.0092570781707764 (1.0441062223337783) + | > current_lr: 5e-06 + | > step_time: 0.3107 (0.3191146765377946) + | > loader_time: 0.0087 (0.0093300753688484) + + + --> TIME: 2024-10-20 19:40:10 -- STEP: 14600/16928 -- GLOBAL_STEP: 14600 + | > loss_text_ce: 0.03411388397216797 (0.035372661597235816) + | > loss_mel_ce: 3.6150922775268555 (4.1401670426701465) + | > loss: 0.9123015403747559 (1.0438849261729697) + | > current_lr: 5e-06 + | > step_time: 0.208 (0.3191323973707979) + | > loader_time: 0.0085 (0.009328261581185692) + + + --> TIME: 2024-10-20 19:40:37 -- STEP: 14650/16928 -- GLOBAL_STEP: 14650 + | > loss_text_ce: 0.034162260591983795 (0.03537126338853165) + | > loss_mel_ce: 3.7128360271453857 (4.139468478944888) + | > loss: 0.9367495775222778 (1.0437099356659463) + | > current_lr: 5e-06 + | > step_time: 0.2984 (0.3191730415780412) + | > loader_time: 0.0091 (0.009326529746983185) + + + --> TIME: 2024-10-20 19:41:04 -- STEP: 14700/16928 -- GLOBAL_STEP: 14700 + | > loss_text_ce: 0.033321212977170944 (0.03536693342435527) + | > loss_mel_ce: 4.0806884765625 (4.138961695035257) + | > loss: 1.0285024642944336 (1.0435821571966433) + | > current_lr: 5e-06 + | > step_time: 0.2998 (0.3191631310813303) + | > loader_time: 0.0091 (0.009324598263721024) + + + --> TIME: 2024-10-20 19:41:31 -- STEP: 14750/16928 -- GLOBAL_STEP: 14750 + | > loss_text_ce: 0.03594209998846054 (0.035365492059517764) + | > loss_mel_ce: 3.856658458709717 (4.13843469873521) + | > loss: 0.9731501340866089 (1.0434500477920157) + | > current_lr: 5e-06 + | > step_time: 0.2256 (0.3191600143626607) + | > loader_time: 0.0086 (0.009322806616960893) + + + --> TIME: 2024-10-20 19:41:56 -- STEP: 14800/16928 -- GLOBAL_STEP: 14800 + | > loss_text_ce: 0.03963169828057289 (0.035363523846215335) + | > loss_mel_ce: 4.225160121917725 (4.137824000780608) + | > loss: 1.0661979913711548 (1.0432968812536556) + | > current_lr: 5e-06 + | > step_time: 0.1704 (0.3191152082423928) + | > loader_time: 0.0082 (0.009320880464605332) + + + --> TIME: 2024-10-20 19:42:22 -- STEP: 14850/16928 -- GLOBAL_STEP: 14850 + | > loss_text_ce: 0.0334501713514328 (0.03536187422044756) + | > loss_mel_ce: 3.6409378051757812 (4.137187772904948) + | > loss: 0.9185969829559326 (1.04313741187054) + | > current_lr: 5e-06 + | > step_time: 0.3094 (0.3190740506817618) + | > loader_time: 0.0083 (0.009318761632899993) + + + --> TIME: 2024-10-20 19:42:50 -- STEP: 14900/16928 -- GLOBAL_STEP: 14900 + | > loss_text_ce: 0.03591252863407135 (0.03535858924556903) + | > loss_mel_ce: 3.915566921234131 (4.1364232359316135) + | > loss: 0.9878698587417603 (1.0429454563928138) + | > current_lr: 5e-06 + | > step_time: 0.3759 (0.3190921646476614) + | > loader_time: 0.0104 (0.009317391270759071) + + + --> TIME: 2024-10-20 19:43:17 -- STEP: 14950/16928 -- GLOBAL_STEP: 14950 + | > loss_text_ce: 0.03614801913499832 (0.03535703486149332) + | > loss_mel_ce: 4.018507480621338 (4.1357475337216325) + | > loss: 1.0136638879776 (1.0427761422351867) + | > current_lr: 5e-06 + | > step_time: 0.3613 (0.3191338994271801) + | > loader_time: 0.0094 (0.009316231542606366) + + + --> TIME: 2024-10-20 19:43:43 -- STEP: 15000/16928 -- GLOBAL_STEP: 15000 + | > loss_text_ce: 0.03598940744996071 (0.0353571932882071) + | > loss_mel_ce: 4.239035606384277 (4.135211389382644) + | > loss: 1.0687562227249146 (1.0426421457250934) + | > current_lr: 5e-06 + | > step_time: 0.1991 (0.31907592883110264) + | > loader_time: 0.0093 (0.009315398136774648) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_15000.pth + + --> TIME: 2024-10-20 19:44:23 -- STEP: 15050/16928 -- GLOBAL_STEP: 15050 + | > loss_text_ce: 0.03587885573506355 (0.03535603504405173) + | > loss_mel_ce: 3.6874287128448486 (4.134361898099031) + | > loss: 0.9308269023895264 (1.0424294833487475) + | > current_lr: 5e-06 + | > step_time: 0.5239 (0.319062132185878) + | > loader_time: 0.0094 (0.009316337354159383) + + + --> TIME: 2024-10-20 19:44:49 -- STEP: 15100/16928 -- GLOBAL_STEP: 15100 + | > loss_text_ce: 0.03093183971941471 (0.03535393700151653) + | > loss_mel_ce: 3.931485414505005 (4.1337556341941415) + | > loss: 0.9906043410301208 (1.042277392883177) + | > current_lr: 5e-06 + | > step_time: 0.2877 (0.3190379075498792) + | > loader_time: 0.0087 (0.00931504214836267) + + + --> TIME: 2024-10-20 19:45:17 -- STEP: 15150/16928 -- GLOBAL_STEP: 15150 + | > loss_text_ce: 0.03376308083534241 (0.035352275340568944) + | > loss_mel_ce: 3.9133355617523193 (4.13333363136439) + | > loss: 0.9867746829986572 (1.042171476780388) + | > current_lr: 5e-06 + | > step_time: 0.2658 (0.3190850965418066) + | > loader_time: 0.0092 (0.00931437111530362) + + + --> TIME: 2024-10-20 19:45:43 -- STEP: 15200/16928 -- GLOBAL_STEP: 15200 + | > loss_text_ce: 0.03676020726561546 (0.03535125483820023) + | > loss_mel_ce: 3.7990522384643555 (4.132590601852029) + | > loss: 0.9589530825614929 (1.041985464272534) + | > current_lr: 5e-06 + | > step_time: 0.1749 (0.3190599474938314) + | > loader_time: 0.0094 (0.009313389956951091) + + + --> TIME: 2024-10-20 19:46:11 -- STEP: 15250/16928 -- GLOBAL_STEP: 15250 + | > loss_text_ce: 0.03705844655632973 (0.035348897746229804) + | > loss_mel_ce: 3.5111396312713623 (4.131854255926382) + | > loss: 0.8870494961738586 (1.0418007885237235) + | > current_lr: 5e-06 + | > step_time: 0.2331 (0.3191019050879578) + | > loader_time: 0.0085 (0.009311767984609084) + + + --> TIME: 2024-10-20 19:46:38 -- STEP: 15300/16928 -- GLOBAL_STEP: 15300 + | > loss_text_ce: 0.03300675004720688 (0.035345318390093056) + | > loss_mel_ce: 4.076745986938477 (4.131384445093785) + | > loss: 1.0274381637573242 (1.041682440954099) + | > current_lr: 5e-06 + | > step_time: 0.2242 (0.3191263186074571) + | > loader_time: 0.0086 (0.009310498798594705) + + + --> TIME: 2024-10-20 19:47:04 -- STEP: 15350/16928 -- GLOBAL_STEP: 15350 + | > loss_text_ce: 0.03460971266031265 (0.035344059483939975) + | > loss_mel_ce: 3.8282110691070557 (4.131004837763003) + | > loss: 0.9657052159309387 (1.041587224406608) + | > current_lr: 5e-06 + | > step_time: 0.3099 (0.31910636504232515) + | > loader_time: 0.0089 (0.009310441281197464) + + + --> TIME: 2024-10-20 19:47:29 -- STEP: 15400/16928 -- GLOBAL_STEP: 15400 + | > loss_text_ce: 0.0326334685087204 (0.03534115622666754) + | > loss_mel_ce: 3.9618711471557617 (4.130390782929053) + | > loss: 0.998626172542572 (1.0414329848738486) + | > current_lr: 5e-06 + | > step_time: 0.2748 (0.31903415848682665) + | > loader_time: 0.0092 (0.009310023428557706) + + + --> TIME: 2024-10-20 19:47:57 -- STEP: 15450/16928 -- GLOBAL_STEP: 15450 + | > loss_text_ce: 0.03641482815146446 (0.0353393150454924) + | > loss_mel_ce: 4.619018077850342 (4.129875968389877) + | > loss: 1.16385817527771 (1.0413038209302548) + | > current_lr: 5e-06 + | > step_time: 0.4365 (0.3190639618993984) + | > loader_time: 0.01 (0.009309852501335219) + + + --> TIME: 2024-10-20 19:48:25 -- STEP: 15500/16928 -- GLOBAL_STEP: 15500 + | > loss_text_ce: 0.03751356154680252 (0.03533700199040679) + | > loss_mel_ce: 4.028014183044434 (4.12938559527547) + | > loss: 1.0163819789886475 (1.0411806493805313) + | > current_lr: 5e-06 + | > step_time: 0.3219 (0.3191375668125789) + | > loader_time: 0.0103 (0.009310086757905973) + + + --> TIME: 2024-10-20 19:48:51 -- STEP: 15550/16928 -- GLOBAL_STEP: 15550 + | > loss_text_ce: 0.03081272356212139 (0.03533480244637786) + | > loss_mel_ce: 4.012628555297852 (4.128845361482633) + | > loss: 1.0108603239059448 (1.041045041030627) + | > current_lr: 5e-06 + | > step_time: 0.2973 (0.3190995452948343) + | > loader_time: 0.0087 (0.009309634917035315) + + + --> TIME: 2024-10-20 19:49:18 -- STEP: 15600/16928 -- GLOBAL_STEP: 15600 + | > loss_text_ce: 0.03698939085006714 (0.03533323900129364) + | > loss_mel_ce: 4.223567485809326 (4.128095769072148) + | > loss: 1.0651391744613647 (1.0408572520468486) + | > current_lr: 5e-06 + | > step_time: 0.3571 (0.31912124652129165) + | > loader_time: 0.0093 (0.009310666597806442) + + + --> TIME: 2024-10-20 19:49:45 -- STEP: 15650/16928 -- GLOBAL_STEP: 15650 + | > loss_text_ce: 0.03288067877292633 (0.03533140413439316) + | > loss_mel_ce: 3.927401065826416 (4.127449906931097) + | > loss: 0.9900704622268677 (1.0406953277831648) + | > current_lr: 5e-06 + | > step_time: 0.4683 (0.31909749300716234) + | > loader_time: 0.0097 (0.00931007967208516) + + + --> TIME: 2024-10-20 19:50:13 -- STEP: 15700/16928 -- GLOBAL_STEP: 15700 + | > loss_text_ce: 0.03577018156647682 (0.03533007683576486) + | > loss_mel_ce: 3.575360059738159 (4.126971818185877) + | > loss: 0.9027825593948364 (1.0405754737755295) + | > current_lr: 5e-06 + | > step_time: 0.2769 (0.3191592943288739) + | > loader_time: 0.0092 (0.009310847756209597) + + + --> TIME: 2024-10-20 19:50:40 -- STEP: 15750/16928 -- GLOBAL_STEP: 15750 + | > loss_text_ce: 0.043121375143527985 (0.035328442431158534) + | > loss_mel_ce: 3.8505704402923584 (4.1261895916650815) + | > loss: 0.9734229445457458 (1.0403795085293908) + | > current_lr: 5e-06 + | > step_time: 0.2324 (0.31917747883569836) + | > loader_time: 0.0095 (0.009310678754533992) + + + --> TIME: 2024-10-20 19:51:09 -- STEP: 15800/16928 -- GLOBAL_STEP: 15800 + | > loss_text_ce: 0.03563596308231354 (0.035326108666773506) + | > loss_mel_ce: 4.663931846618652 (4.125603163679908) + | > loss: 1.1748919486999512 (1.0402323180814335) + | > current_lr: 5e-06 + | > step_time: 0.2063 (0.3192413753044777) + | > loader_time: 0.0089 (0.0093104594115969) + + + --> TIME: 2024-10-20 19:51:35 -- STEP: 15850/16928 -- GLOBAL_STEP: 15850 + | > loss_text_ce: 0.034753553569316864 (0.03532382021273259) + | > loss_mel_ce: 4.058280944824219 (4.124915177619006) + | > loss: 1.0232585668563843 (1.0400597494520978) + | > current_lr: 5e-06 + | > step_time: 0.2871 (0.3192341594515559) + | > loader_time: 0.009 (0.009310125868403149) + + + --> TIME: 2024-10-20 19:52:01 -- STEP: 15900/16928 -- GLOBAL_STEP: 15900 + | > loss_text_ce: 0.0338139608502388 (0.03532119587380092) + | > loss_mel_ce: 3.363499164581299 (4.12438902042193) + | > loss: 0.8493282794952393 (1.0399275540668262) + | > current_lr: 5e-06 + | > step_time: 0.1885 (0.31919389910668156) + | > loader_time: 0.0082 (0.009310219647749366) + + + --> TIME: 2024-10-20 19:52:27 -- STEP: 15950/16928 -- GLOBAL_STEP: 15950 + | > loss_text_ce: 0.03838656470179558 (0.03532050260252157) + | > loss_mel_ce: 4.787449359893799 (4.12383294736326) + | > loss: 1.2064589262008667 (1.0397883624866093) + | > current_lr: 5e-06 + | > step_time: 0.3338 (0.3191791752961756) + | > loader_time: 0.0099 (0.00931017019158242) + + + --> TIME: 2024-10-20 19:52:54 -- STEP: 16000/16928 -- GLOBAL_STEP: 16000 + | > loss_text_ce: 0.0330905020236969 (0.035321009067585736) + | > loss_mel_ce: 3.7183854579925537 (4.123155299052556) + | > loss: 0.9378690123558044 (1.0396190770417455) + | > current_lr: 5e-06 + | > step_time: 0.2106 (0.3191860599666854) + | > loader_time: 0.0086 (0.009309752359986257) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_16000.pth + + --> TIME: 2024-10-20 19:53:32 -- STEP: 16050/16928 -- GLOBAL_STEP: 16050 + | > loss_text_ce: 0.03865858539938927 (0.035321120772615254) + | > loss_mel_ce: 4.511946678161621 (4.1224530027811) + | > loss: 1.1376513242721558 (1.039443530898971) + | > current_lr: 5e-06 + | > step_time: 0.2302 (0.31910576419295755) + | > loader_time: 0.0087 (0.009310888011136111) + + + --> TIME: 2024-10-20 19:53:59 -- STEP: 16100/16928 -- GLOBAL_STEP: 16100 + | > loss_text_ce: 0.03633442893624306 (0.03531975962857934) + | > loss_mel_ce: 3.326917886734009 (4.121957991182394) + | > loss: 0.8408131003379822 (1.0393194377126163) + | > current_lr: 5e-06 + | > step_time: 0.2437 (0.31910753458923635) + | > loader_time: 0.0085 (0.009310730688320137) + + + --> TIME: 2024-10-20 19:54:26 -- STEP: 16150/16928 -- GLOBAL_STEP: 16150 + | > loss_text_ce: 0.034952327609062195 (0.035317500039543016) + | > loss_mel_ce: 3.7754709720611572 (4.121237753035444) + | > loss: 0.9526058435440063 (1.0391388132808381) + | > current_lr: 5e-06 + | > step_time: 0.4978 (0.31911895422743775) + | > loader_time: 0.0102 (0.00931044888570208) + + + --> TIME: 2024-10-20 19:54:55 -- STEP: 16200/16928 -- GLOBAL_STEP: 16200 + | > loss_text_ce: 0.037680964916944504 (0.03531626069665327) + | > loss_mel_ce: 4.262294292449951 (4.120559111038804) + | > loss: 1.0749938488006592 (1.0389688429420378) + | > current_lr: 5e-06 + | > step_time: 0.2545 (0.31920215656728124) + | > loader_time: 0.0094 (0.009310626645147018) + + + --> TIME: 2024-10-20 19:55:20 -- STEP: 16250/16928 -- GLOBAL_STEP: 16250 + | > loss_text_ce: 0.042309682816267014 (0.035316171790888716) + | > loss_mel_ce: 3.9673287868499756 (4.119770168700546) + | > loss: 1.00240957736969 (1.0387715851563677) + | > current_lr: 5e-06 + | > step_time: 0.3411 (0.3191437263635509) + | > loader_time: 0.0104 (0.009310296792250365) + + + --> TIME: 2024-10-20 19:55:47 -- STEP: 16300/16928 -- GLOBAL_STEP: 16300 + | > loss_text_ce: 0.0371839702129364 (0.035313893424038546) + | > loss_mel_ce: 4.471995830535889 (4.11922020210075) + | > loss: 1.127294898033142 (1.0386335239198312) + | > current_lr: 5e-06 + | > step_time: 0.3056 (0.319151394060055) + | > loader_time: 0.0102 (0.009310201018865807) + + + --> TIME: 2024-10-20 19:56:12 -- STEP: 16350/16928 -- GLOBAL_STEP: 16350 + | > loss_text_ce: 0.035342250019311905 (0.03531006589190133) + | > loss_mel_ce: 3.723151922225952 (4.1184982478873415) + | > loss: 0.9396235346794128 (1.0384520784685738) + | > current_lr: 5e-06 + | > step_time: 0.3004 (0.31908731976780524) + | > loader_time: 0.01 (0.009310008553554476) + + + --> TIME: 2024-10-20 19:56:39 -- STEP: 16400/16928 -- GLOBAL_STEP: 16400 + | > loss_text_ce: 0.03552352264523506 (0.03530832196733487) + | > loss_mel_ce: 3.56962513923645 (4.118135417641624) + | > loss: 0.9012871384620667 (1.0383609349189737) + | > current_lr: 5e-06 + | > step_time: 0.3197 (0.3190707570459804) + | > loader_time: 0.0094 (0.009309968279629172) + + + --> TIME: 2024-10-20 19:57:07 -- STEP: 16450/16928 -- GLOBAL_STEP: 16450 + | > loss_text_ce: 0.033930033445358276 (0.035305936767586685) + | > loss_mel_ce: 4.766165256500244 (4.1175427302908) + | > loss: 1.2000237703323364 (1.038212166781846) + | > current_lr: 5e-06 + | > step_time: 0.4331 (0.3191272237308139) + | > loader_time: 0.0095 (0.009309649279052317) + + + --> TIME: 2024-10-20 19:57:35 -- STEP: 16500/16928 -- GLOBAL_STEP: 16500 + | > loss_text_ce: 0.03335106000304222 (0.03530364719597683) + | > loss_mel_ce: 4.140079021453857 (4.117115253795237) + | > loss: 1.043357491493225 (1.0381047252741729) + | > current_lr: 5e-06 + | > step_time: 0.2448 (0.31919268109581883) + | > loader_time: 0.0095 (0.009310593359398073) + + + --> TIME: 2024-10-20 19:58:02 -- STEP: 16550/16928 -- GLOBAL_STEP: 16550 + | > loss_text_ce: 0.03394843637943268 (0.03530061556459648) + | > loss_mel_ce: 3.7315828800201416 (4.11642525789597) + | > loss: 0.9413828253746033 (1.037931468411155) + | > current_lr: 5e-06 + | > step_time: 0.3421 (0.31920759461797815) + | > loader_time: 0.0091 (0.009310742101640453) + + + --> TIME: 2024-10-20 19:58:29 -- STEP: 16600/16928 -- GLOBAL_STEP: 16600 + | > loss_text_ce: 0.034961339086294174 (0.03529811232002927) + | > loss_mel_ce: 3.992586851119995 (4.115684920477541) + | > loss: 1.0068870782852173 (1.0377457582483816) + | > current_lr: 5e-06 + | > step_time: 0.3993 (0.3192118612398604) + | > loader_time: 0.0102 (0.009310888497226163) + + + --> TIME: 2024-10-20 19:58:57 -- STEP: 16650/16928 -- GLOBAL_STEP: 16650 + | > loss_text_ce: 0.038126926869153976 (0.0352986055036431) + | > loss_mel_ce: 4.065340042114258 (4.115199252678429) + | > loss: 1.0258667469024658 (1.0376244645934924) + | > current_lr: 5e-06 + | > step_time: 0.4687 (0.3192507312390917) + | > loader_time: 0.0092 (0.009310973284838747) + + + --> TIME: 2024-10-20 19:59:25 -- STEP: 16700/16928 -- GLOBAL_STEP: 16700 + | > loss_text_ce: 0.03602786362171173 (0.03529728989914332) + | > loss_mel_ce: 3.8913285732269287 (4.114650801755673) + | > loss: 0.981839120388031 (1.0374870229659685) + | > current_lr: 5e-06 + | > step_time: 0.1736 (0.31927904574457233) + | > loader_time: 0.0085 (0.009311264660543979) + + + --> TIME: 2024-10-20 19:59:53 -- STEP: 16750/16928 -- GLOBAL_STEP: 16750 + | > loss_text_ce: 0.033174581825733185 (0.03529565842427429) + | > loss_mel_ce: 4.052950859069824 (4.113942552908106) + | > loss: 1.021531343460083 (1.0373095528830345) + | > current_lr: 5e-06 + | > step_time: 0.2621 (0.3193601727556845) + | > loader_time: 0.0087 (0.009311746398014764) + + + --> TIME: 2024-10-20 20:00:20 -- STEP: 16800/16928 -- GLOBAL_STEP: 16800 + | > loss_text_ce: 0.03726520761847496 (0.0352926765457683) + | > loss_mel_ce: 4.410891056060791 (4.113534561282076) + | > loss: 1.1120390892028809 (1.037206809488791) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.3193345308587683) + | > loader_time: 0.0093 (0.00931175991183231) + + + --> TIME: 2024-10-20 20:00:46 -- STEP: 16850/16928 -- GLOBAL_STEP: 16850 + | > loss_text_ce: 0.03163515403866768 (0.03529162054964365) + | > loss_mel_ce: 4.1166672706604 (4.112830182621633) + | > loss: 1.0370756387710571 (1.0370304508202162) + | > current_lr: 5e-06 + | > step_time: 0.491 (0.3193125350397508) + | > loader_time: 0.0099 (0.009311232863972334) + + + --> TIME: 2024-10-20 20:01:11 -- STEP: 16900/16928 -- GLOBAL_STEP: 16900 + | > loss_text_ce: 0.03143291547894478 (0.035289808648291295) + | > loss_mel_ce: 4.011976718902588 (4.112294199240948) + | > loss: 1.0108524560928345 (1.036896001999901) + | > current_lr: 5e-06 + | > step_time: 0.3483 (0.3192605154330932) + | > loader_time: 0.0092 (0.009310598246444564) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.39677436551647977 (+0) + | > avg_loss_text_ce: 0.033466432383005076 (+0) + | > avg_loss_mel_ce: 3.7548906974449845 (+0) + | > avg_loss: 3.788357129353963 (+0) + + > BEST MODEL : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/best_model_16928.pth + + > EPOCH: 1/5 + --> /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b + + > TRAINING (2024-10-20 20:12:52)  + + --> TIME: 2024-10-20 20:13:05 -- STEP: 22/16928 -- GLOBAL_STEP: 16950 + | > loss_text_ce: 0.033109746873378754 (0.03372327949513088) + | > loss_mel_ce: 4.075105667114258 (3.8729659644040195) + | > loss: 1.0270538330078125 (0.9766722944649783) + | > current_lr: 5e-06 + | > step_time: 0.2615 (0.30053540793332184) + | > loader_time: 0.0098 (0.010038310831243341) + + + --> TIME: 2024-10-20 20:13:31 -- STEP: 72/16928 -- GLOBAL_STEP: 17000 + | > loss_text_ce: 0.031595513224601746 (0.03432003002510302) + | > loss_mel_ce: 4.333611488342285 (3.870693302816815) + | > loss: 1.0913017988204956 (0.9762533315353923) + | > current_lr: 5e-06 + | > step_time: 0.2083 (0.31184892521964175) + | > loader_time: 0.0092 (0.009736733304129705) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_17000.pth + + --> TIME: 2024-10-20 20:14:08 -- STEP: 122/16928 -- GLOBAL_STEP: 17050 + | > loss_text_ce: 0.03450673446059227 (0.03455810800011529) + | > loss_mel_ce: 3.92649507522583 (3.8817404879898323) + | > loss: 0.9902504682540894 (0.9790746476806578) + | > current_lr: 5e-06 + | > step_time: 0.2465 (0.3115644083648431) + | > loader_time: 0.0089 (0.009802871063107347) + + + --> TIME: 2024-10-20 20:14:34 -- STEP: 172/16928 -- GLOBAL_STEP: 17100 + | > loss_text_ce: 0.03426729887723923 (0.0345818299128739) + | > loss_mel_ce: 3.697842597961426 (3.8886771160502764) + | > loss: 0.933027446269989 (0.980814736249835) + | > current_lr: 5e-06 + | > step_time: 0.1509 (0.3117578195971113) + | > loader_time: 0.0083 (0.009802786416785662) + + + --> TIME: 2024-10-20 20:15:01 -- STEP: 222/16928 -- GLOBAL_STEP: 17150 + | > loss_text_ce: 0.03497143089771271 (0.0347142476789855) + | > loss_mel_ce: 3.615694522857666 (3.8930694627332256) + | > loss: 0.9126664996147156 (0.9819459273471488) + | > current_lr: 5e-06 + | > step_time: 0.3125 (0.31521757014162904) + | > loader_time: 0.0091 (0.00971684369954977) + + + --> TIME: 2024-10-20 20:15:29 -- STEP: 272/16928 -- GLOBAL_STEP: 17200 + | > loss_text_ce: 0.03686597943305969 (0.034699449630673354) + | > loss_mel_ce: 4.21699333190918 (3.893330983379308) + | > loss: 1.063464879989624 (0.9820076075985151) + | > current_lr: 5e-06 + | > step_time: 0.1545 (0.31858862410573396) + | > loader_time: 0.0102 (0.009667574482805589) + + + --> TIME: 2024-10-20 20:15:55 -- STEP: 322/16928 -- GLOBAL_STEP: 17250 + | > loss_text_ce: 0.038163747638463974 (0.034816056727113597) + | > loss_mel_ce: 3.905236005783081 (3.8899224855885004) + | > loss: 0.985849916934967 (0.981184635103119) + | > current_lr: 5e-06 + | > step_time: 0.4312 (0.3171024855619633) + | > loader_time: 0.0099 (0.009610933546694171) + + + --> TIME: 2024-10-20 20:16:22 -- STEP: 372/16928 -- GLOBAL_STEP: 17300 + | > loss_text_ce: 0.03371058404445648 (0.03481746593650475) + | > loss_mel_ce: 3.629366636276245 (3.8931594414095723) + | > loss: 0.9157692790031433 (0.9819942267351253) + | > current_lr: 5e-06 + | > step_time: 0.2742 (0.3185801743179238) + | > loader_time: 0.0094 (0.009590723822193765) + + + --> TIME: 2024-10-20 20:16:49 -- STEP: 422/16928 -- GLOBAL_STEP: 17350 + | > loss_text_ce: 0.03651593253016472 (0.034843804822325464) + | > loss_mel_ce: 3.7272467613220215 (3.8886018313502815) + | > loss: 0.9409406781196594 (0.9808614081681057) + | > current_lr: 5e-06 + | > step_time: 0.2144 (0.318785673634136) + | > loader_time: 0.011 (0.00957663985790235) + + + --> TIME: 2024-10-20 20:17:16 -- STEP: 472/16928 -- GLOBAL_STEP: 17400 + | > loss_text_ce: 0.03473854809999466 (0.03481634538541788) + | > loss_mel_ce: 3.8369245529174805 (3.8904584499738983) + | > loss: 0.9679157733917236 (0.9813186978384599) + | > current_lr: 5e-06 + | > step_time: 0.2455 (0.31823897058680917) + | > loader_time: 0.0095 (0.009584302619352184) + + + --> TIME: 2024-10-20 20:17:43 -- STEP: 522/16928 -- GLOBAL_STEP: 17450 + | > loss_text_ce: 0.035858143121004105 (0.03481625980016033) + | > loss_mel_ce: 3.338716745376587 (3.8920358044891064) + | > loss: 0.8436437249183655 (0.9817130147725687) + | > current_lr: 5e-06 + | > step_time: 0.3618 (0.31951366621872485) + | > loader_time: 0.0103 (0.00956561282220015) + + + --> TIME: 2024-10-20 20:18:10 -- STEP: 572/16928 -- GLOBAL_STEP: 17500 + | > loss_text_ce: 0.03223782777786255 (0.034789737359575065) + | > loss_mel_ce: 4.801973342895508 (3.8990440885503808) + | > loss: 1.2085528373718262 (0.9834584546881122) + | > current_lr: 5e-06 + | > step_time: 0.2743 (0.31954559204461713) + | > loader_time: 0.0085 (0.009550524341476548) + + + --> TIME: 2024-10-20 20:18:37 -- STEP: 622/16928 -- GLOBAL_STEP: 17550 + | > loss_text_ce: 0.03504408895969391 (0.034779238065411264) + | > loss_mel_ce: 3.7736353874206543 (3.9012487333303865) + | > loss: 0.9521698951721191 (0.9840069914362438) + | > current_lr: 5e-06 + | > step_time: 0.3591 (0.3194549478518622) + | > loader_time: 0.0094 (0.009558347643763307) + + + --> TIME: 2024-10-20 20:19:02 -- STEP: 672/16928 -- GLOBAL_STEP: 17600 + | > loss_text_ce: 0.03451113775372505 (0.03476090607277699) + | > loss_mel_ce: 4.131163120269775 (3.9002151208974065) + | > loss: 1.0414185523986816 (0.9837440055395875) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.31826037416855535) + | > loader_time: 0.0091 (0.009532470433484935) + + + --> TIME: 2024-10-20 20:19:29 -- STEP: 722/16928 -- GLOBAL_STEP: 17650 + | > loss_text_ce: 0.034133754670619965 (0.03475667549673875) + | > loss_mel_ce: 3.7821478843688965 (3.894728743798845) + | > loss: 0.9540703892707825 (0.9823713538719346) + | > current_lr: 5e-06 + | > step_time: 0.4356 (0.31813902603952526) + | > loader_time: 0.01 (0.009526690617822861) + + + --> TIME: 2024-10-20 20:19:56 -- STEP: 772/16928 -- GLOBAL_STEP: 17700 + | > loss_text_ce: 0.03562498092651367 (0.03476434591868075) + | > loss_mel_ce: 3.5347506999969482 (3.8964389201272955) + | > loss: 0.8925939202308655 (0.9828008157756044) + | > current_lr: 5e-06 + | > step_time: 0.2043 (0.31886335265451166) + | > loader_time: 0.009 (0.009518861770629876) + + + --> TIME: 2024-10-20 20:20:22 -- STEP: 822/16928 -- GLOBAL_STEP: 17750 + | > loss_text_ce: 0.03285469114780426 (0.03473455363254625) + | > loss_mel_ce: 4.109686851501465 (3.8992199143644086) + | > loss: 1.0356353521347046 (0.9834886160645172) + | > current_lr: 5e-06 + | > step_time: 0.4319 (0.31828218045896) + | > loader_time: 0.0096 (0.009515755077928225) + + + --> TIME: 2024-10-20 20:20:50 -- STEP: 872/16928 -- GLOBAL_STEP: 17800 + | > loss_text_ce: 0.03147066757082939 (0.03472660500727118) + | > loss_mel_ce: 3.564413547515869 (3.90231623698812) + | > loss: 0.8989710807800293 (0.9842607098708459) + | > current_lr: 5e-06 + | > step_time: 0.2714 (0.31929357636959194) + | > loader_time: 0.0095 (0.0095120703955309) + + + --> TIME: 2024-10-20 20:21:16 -- STEP: 922/16928 -- GLOBAL_STEP: 17850 + | > loss_text_ce: 0.034437693655490875 (0.03470603301474696) + | > loss_mel_ce: 3.827979803085327 (3.903884172956752) + | > loss: 0.9656043648719788 (0.9846475509655966) + | > current_lr: 5e-06 + | > step_time: 0.3 (0.31882761797005094) + | > loader_time: 0.0084 (0.009501539445492285) + + + --> TIME: 2024-10-20 20:21:43 -- STEP: 972/16928 -- GLOBAL_STEP: 17900 + | > loss_text_ce: 0.03373830392956734 (0.034709392600298095) + | > loss_mel_ce: 4.4531145095825195 (3.899971764274095) + | > loss: 1.1217131614685059 (0.9836702887658719) + | > current_lr: 5e-06 + | > step_time: 0.2715 (0.3185815575682088) + | > loader_time: 0.0092 (0.009497510307610271) + + + --> TIME: 2024-10-20 20:22:09 -- STEP: 1022/16928 -- GLOBAL_STEP: 17950 + | > loss_text_ce: 0.03421369194984436 (0.034708192648544745) + | > loss_mel_ce: 4.552002429962158 (3.8982346092884788) + | > loss: 1.1465539932250977 (0.983235700023151) + | > current_lr: 5e-06 + | > step_time: 0.3769 (0.31844985695034567) + | > loader_time: 0.0097 (0.009487462370362992) + + + --> TIME: 2024-10-20 20:22:36 -- STEP: 1072/16928 -- GLOBAL_STEP: 18000 + | > loss_text_ce: 0.036879874765872955 (0.034699496121576745) + | > loss_mel_ce: 4.028604984283447 (3.896914698961955) + | > loss: 1.016371250152588 (0.9829035483634294) + | > current_lr: 5e-06 + | > step_time: 0.2232 (0.31846759412715686) + | > loader_time: 0.0083 (0.00946773255049292) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b/checkpoint_18000.pth + + --> TIME: 2024-10-20 20:23:13 -- STEP: 1122/16928 -- GLOBAL_STEP: 18050 + | > loss_text_ce: 0.0327695794403553 (0.03470168040597187) + | > loss_mel_ce: 4.388420104980469 (3.896036397964559) + | > loss: 1.1052974462509155 (0.9826845193079237) + | > current_lr: 5e-06 + | > step_time: 0.3471 (0.31877245431277845) + | > loader_time: 0.0098 (0.009463138971311727) + + + --> TIME: 2024-10-20 20:23:39 -- STEP: 1172/16928 -- GLOBAL_STEP: 18100 + | > loss_text_ce: 0.03331354632973671 (0.034719247277190377) + | > loss_mel_ce: 3.7322261333465576 (3.8963919430462575) + | > loss: 0.9413849115371704 (0.9827777973722679) + | > current_lr: 5e-06 + | > step_time: 0.3485 (0.31827887740151484) + | > loader_time: 0.0091 (0.009443694820989923) + + + --> TIME: 2024-10-20 20:24:05 -- STEP: 1222/16928 -- GLOBAL_STEP: 18150 + | > loss_text_ce: 0.03618785738945007 (0.03471847637762269) + | > loss_mel_ce: 4.182468414306641 (3.8990944042533586) + | > loss: 1.0546640157699585 (0.9834532195339422) + | > current_lr: 5e-06 + | > step_time: 0.4657 (0.317678229898556) + | > loader_time: 0.009 (0.009432318557889093) + + + --> TIME: 2024-10-20 20:24:31 -- STEP: 1272/16928 -- GLOBAL_STEP: 18200 + | > loss_text_ce: 0.037352561950683594 (0.03472578262693527) + | > loss_mel_ce: 3.4993271827697754 (3.8976375393147737) + | > loss: 0.8841699361801147 (0.9830908298304996) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.3173512445305879) + | > loader_time: 0.0081 (0.009398992706394788) + + + --> TIME: 2024-10-20 20:24:57 -- STEP: 1322/16928 -- GLOBAL_STEP: 18250 + | > loss_text_ce: 0.035484857857227325 (0.034733069966210485) + | > loss_mel_ce: 3.793217658996582 (3.899339139732038) + | > loss: 0.957175612449646 (0.9835180518937363) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.317448606649794) + | > loader_time: 0.0088 (0.00937643318060847) + + + --> TIME: 2024-10-20 20:25:25 -- STEP: 1372/16928 -- GLOBAL_STEP: 18300 + | > loss_text_ce: 0.0378490574657917 (0.034739390542356016) + | > loss_mel_ce: 4.092213153839111 (3.9008624167901087) + | > loss: 1.032515525817871 (0.983900451321296) + | > current_lr: 5e-06 + | > step_time: 0.2738 (0.31783227496522487) + | > loader_time: 0.0098 (0.009351382276407123) + + + --> TIME: 2024-10-20 20:25:51 -- STEP: 1422/16928 -- GLOBAL_STEP: 18350 + | > loss_text_ce: 0.033935680985450745 (0.034754963434509616) + | > loss_mel_ce: 3.8142991065979004 (3.9015297366596977) + | > loss: 0.9620587229728699 (0.98407117467054) + | > current_lr: 5e-06 + | > step_time: 0.3355 (0.31789568573781346) + | > loader_time: 0.0086 (0.009329206832853534) + + + --> TIME: 2024-10-20 20:26:18 -- STEP: 1472/16928 -- GLOBAL_STEP: 18400 + | > loss_text_ce: 0.03335033729672432 (0.03475578199646884) + | > loss_mel_ce: 3.9329395294189453 (3.9021651433861773) + | > loss: 0.9915724396705627 (0.9842302311292809) + | > current_lr: 5e-06 + | > step_time: 0.3458 (0.317938111884438) + | > loader_time: 0.0089 (0.009300437634405861) + + + --> TIME: 2024-10-20 20:26:45 -- STEP: 1522/16928 -- GLOBAL_STEP: 18450 + | > loss_text_ce: 0.03268079087138176 (0.034752102047400805) + | > loss_mel_ce: 4.1107940673828125 (3.9025505876729114) + | > loss: 1.035868763923645 (0.98432567241784) + | > current_lr: 5e-06 + | > step_time: 0.2545 (0.3178411397921424) + | > loader_time: 0.009 (0.009288152006703186) + + + --> TIME: 2024-10-20 20:27:12 -- STEP: 1572/16928 -- GLOBAL_STEP: 18500 + | > loss_text_ce: 0.03480798006057739 (0.03475833372935733) + | > loss_mel_ce: 4.534967422485352 (3.904892966644151) + | > loss: 1.1424438953399658 (0.9849128251464009) + | > current_lr: 5e-06 + | > step_time: 0.248 (0.31792777928383853) + | > loader_time: 0.0088 (0.009271626860737481) + + + --> TIME: 2024-10-20 20:27:39 -- STEP: 1622/16928 -- GLOBAL_STEP: 18550 + | > loss_text_ce: 0.03341677412390709 (0.03475374784136858) + | > loss_mel_ce: 3.915621280670166 (3.9062706164455294) + | > loss: 0.9872595071792603 (0.9852560910725858) + | > current_lr: 5e-06 + | > step_time: 0.4691 (0.31808271405141214) + | > loader_time: 0.0097 (0.009256351743761602) + + + --> TIME: 2024-10-20 20:28:05 -- STEP: 1672/16928 -- GLOBAL_STEP: 18600 + | > loss_text_ce: 0.03329416736960411 (0.03473707457723622) + | > loss_mel_ce: 4.1293134689331055 (3.905930786897121) + | > loss: 1.0406519174575806 (0.9851669653109386) + | > current_lr: 5e-06 + | > step_time: 0.2906 (0.31821737021350377) + | > loader_time: 0.0089 (0.009263982898310593) + + + --> TIME: 2024-10-20 20:28:32 -- STEP: 1722/16928 -- GLOBAL_STEP: 18650 + | > loss_text_ce: 0.038450051099061966 (0.03474344698870154) + | > loss_mel_ce: 3.8487768173217773 (3.9055343682203834) + | > loss: 0.9718067049980164 (0.9850694536470509) + | > current_lr: 5e-06 + | > step_time: 0.1924 (0.31831378139269867) + | > loader_time: 0.009 (0.009265654178446572) + + + --> TIME: 2024-10-20 20:28:59 -- STEP: 1772/16928 -- GLOBAL_STEP: 18700 + | > loss_text_ce: 0.033860836178064346 (0.034728869602386886) + | > loss_mel_ce: 4.314152240753174 (3.9037139246210826) + | > loss: 1.087003231048584 (0.9846106982930788) + | > current_lr: 5e-06 + | > step_time: 0.2254 (0.31814720601045227) + | > loader_time: 0.0094 (0.009267505754466497) + + ! Run is kept in /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_05+26PM-1a53d9b