diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,6015 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 4 + | > Num. of Torch Threads: 1 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=/kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b + + > Model has 522165080 parameters + + > EPOCH: 0/5 + --> /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b + + > TRAINING (2024-10-20 04:25:28)  + + --> TIME: 2024-10-20 04:25:32 -- STEP: 0/16928 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.0391542911529541 (0.0391542911529541) + | > loss_mel_ce: 4.246133327484131 (4.246133327484131) + | > loss: 1.071321964263916 (1.071321964263916) + | > current_lr: 5e-06 + | > step_time: 0.664 (0.664008378982544) + | > loader_time: 3.6126 (3.6126058101654053) + + + --> TIME: 2024-10-20 04:25:59 -- STEP: 50/16928 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.04044220224022865 (0.04304232805967331) + | > loss_mel_ce: 4.386104106903076 (4.3178960657119765) + | > loss: 1.1066365242004395 (1.0902346003055576) + | > current_lr: 5e-06 + | > step_time: 0.3376 (0.32467098712921133) + | > loader_time: 0.0147 (0.014261317253112794) + + + --> TIME: 2024-10-20 04:26:29 -- STEP: 100/16928 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.04135171324014664 (0.04326200932264328) + | > loss_mel_ce: 4.374904155731201 (4.270129253864291) + | > loss: 1.1040639877319336 (1.078347818851471) + | > current_lr: 5e-06 + | > step_time: 0.3543 (0.33842167854309063) + | > loader_time: 0.0155 (0.014326789379119874) + + + --> TIME: 2024-10-20 04:26:55 -- STEP: 150/16928 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.040148548781871796 (0.04333433106541636) + | > loss_mel_ce: 4.406633377075195 (4.202515077590944) + | > loss: 1.1116955280303955 (1.0614623542626695) + | > current_lr: 5e-06 + | > step_time: 0.2621 (0.329479826291402) + | > loader_time: 0.0133 (0.014130775133768718) + + + --> TIME: 2024-10-20 04:27:21 -- STEP: 200/16928 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.04360450804233551 (0.0435130824521184) + | > loss_mel_ce: 4.186448097229004 (4.161222952604294) + | > loss: 1.0575131177902222 (1.0511840105056762) + | > current_lr: 5e-06 + | > step_time: 0.2734 (0.32251884102821343) + | > loader_time: 0.0104 (0.013838365077972412) + + + --> TIME: 2024-10-20 04:27:48 -- STEP: 250/16928 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.0435061939060688 (0.04348920020461083) + | > loss_mel_ce: 4.0520477294921875 (4.141283948898315) + | > loss: 1.0238884687423706 (1.0461932892799386) + | > current_lr: 5e-06 + | > step_time: 0.2952 (0.32210820674896234) + | > loader_time: 0.0115 (0.013718978881835937) + + + --> TIME: 2024-10-20 04:28:16 -- STEP: 300/16928 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.04461733251810074 (0.04347549043595792) + | > loss_mel_ce: 3.601017713546753 (4.1223645885785425) + | > loss: 0.9114087820053101 (1.0414600219329209) + | > current_lr: 5e-06 + | > step_time: 0.2435 (0.32523251056671126) + | > loader_time: 0.0088 (0.013609696229298916) + + + --> TIME: 2024-10-20 04:28:44 -- STEP: 350/16928 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.04410836473107338 (0.04328271632747993) + | > loss_mel_ce: 4.0746989250183105 (4.098186524936129) + | > loss: 1.029701828956604 (1.0353673122610383) + | > current_lr: 5e-06 + | > step_time: 0.3793 (0.3252631255558559) + | > loader_time: 0.0095 (0.013416139738900325) + + + --> TIME: 2024-10-20 04:29:10 -- STEP: 400/16928 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.041675809770822525 (0.043206824110820934) + | > loss_mel_ce: 3.71531081199646 (4.08500513672828) + | > loss: 0.939246654510498 (1.032052992135288) + | > current_lr: 5e-06 + | > step_time: 0.2443 (0.3231691247224808) + | > loader_time: 0.0095 (0.013193382620811466) + + + --> TIME: 2024-10-20 04:29:35 -- STEP: 450/16928 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.044594839215278625 (0.04312915815247434) + | > loss_mel_ce: 3.6861767768859863 (4.077611077096722) + | > loss: 0.9326928853988647 (1.0301850608984628) + | > current_lr: 5e-06 + | > step_time: 0.3791 (0.32060581578148745) + | > loader_time: 0.0147 (0.012992005348205571) + + + --> TIME: 2024-10-20 04:30:03 -- STEP: 500/16928 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.04473448544740677 (0.043041453391313586) + | > loss_mel_ce: 3.8658061027526855 (4.0680785727500846) + | > loss: 0.9776351451873779 (1.0277800086736681) + | > current_lr: 5e-06 + | > step_time: 0.2709 (0.32173923397064214) + | > loader_time: 0.0093 (0.012801203250885013) + + + --> TIME: 2024-10-20 04:30:30 -- STEP: 550/16928 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.03989434987306595 (0.04306225297125906) + | > loss_mel_ce: 4.471467971801758 (4.066081395149225) + | > loss: 1.1278406381607056 (1.0272859142043378) + | > current_lr: 5e-06 + | > step_time: 0.2521 (0.32179569981314915) + | > loader_time: 0.0105 (0.012672885547984734) + + + --> TIME: 2024-10-20 04:30:57 -- STEP: 600/16928 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.04339165240526199 (0.04308243031303092) + | > loss_mel_ce: 3.8176109790802 (4.058076386451717) + | > loss: 0.9652506709098816 (1.0252897056937225) + | > current_lr: 5e-06 + | > step_time: 0.2747 (0.3215464234352109) + | > loader_time: 0.0102 (0.012565944592158011) + + + --> TIME: 2024-10-20 04:31:24 -- STEP: 650/16928 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.03905012086033821 (0.04311530801539242) + | > loss_mel_ce: 3.6192212104797363 (4.04595671873826) + | > loss: 0.9145678281784058 (1.0222680085438953) + | > current_lr: 5e-06 + | > step_time: 0.3044 (0.3220463268573467) + | > loader_time: 0.009 (0.012454505700331478) + + + --> TIME: 2024-10-20 04:31:53 -- STEP: 700/16928 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.039684269577264786 (0.04308036625917472) + | > loss_mel_ce: 4.104108810424805 (4.04229875053678) + | > loss: 1.0359482765197754 (1.021344780496189) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.323234530857631) + | > loader_time: 0.0095 (0.012338109357016437) + + + --> TIME: 2024-10-20 04:32:21 -- STEP: 750/16928 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.04370790719985962 (0.04305230282247068) + | > loss_mel_ce: 3.887606382369995 (4.038403485298153) + | > loss: 0.9828285574913025 (1.0203639481067668) + | > current_lr: 5e-06 + | > step_time: 0.3563 (0.3239010187784828) + | > loader_time: 0.0176 (0.012248471895853688) + + + --> TIME: 2024-10-20 04:32:47 -- STEP: 800/16928 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.043663982301950455 (0.04302279974799605) + | > loss_mel_ce: 4.124500274658203 (4.0358250820636705) + | > loss: 1.0420410633087158 (1.0197119715809828) + | > current_lr: 5e-06 + | > step_time: 0.2736 (0.32306118339300144) + | > loader_time: 0.0096 (0.012139532268047341) + + + --> TIME: 2024-10-20 04:33:15 -- STEP: 850/16928 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.04097069799900055 (0.04302916921675207) + | > loss_mel_ce: 3.742534637451172 (4.030348638646741) + | > loss: 0.9458763599395752 (1.018344453152489) + | > current_lr: 5e-06 + | > step_time: 0.288 (0.3241591498431036) + | > loader_time: 0.011 (0.012051479676190554) + + + --> TIME: 2024-10-20 04:33:42 -- STEP: 900/16928 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.03951038792729378 (0.04296928961243896) + | > loss_mel_ce: 4.039737224578857 (4.023001867400275) + | > loss: 1.0198118686676025 (1.0164927900499778) + | > current_lr: 5e-06 + | > step_time: 0.2493 (0.3237349298265244) + | > loader_time: 0.0091 (0.011982015768686937) + + + --> TIME: 2024-10-20 04:34:09 -- STEP: 950/16928 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.04755307734012604 (0.04296657666563989) + | > loss_mel_ce: 3.660573720932007 (4.018398206610428) + | > loss: 0.9270316958427429 (1.0153411966248573) + | > current_lr: 5e-06 + | > step_time: 0.4682 (0.32367864884828257) + | > loader_time: 0.0099 (0.011913104308278945) + + + --> TIME: 2024-10-20 04:34:37 -- STEP: 1000/16928 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.042349688708782196 (0.04293610344827176) + | > loss_mel_ce: 3.740025520324707 (4.018316037654878) + | > loss: 0.9455937743186951 (1.0153130360245723) + | > current_lr: 5e-06 + | > step_time: 0.3941 (0.32432836055755604) + | > loader_time: 0.0112 (0.011842980384826667) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_1000.pth + + --> TIME: 2024-10-20 04:35:13 -- STEP: 1050/16928 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.04550768435001373 (0.04290265544894197) + | > loss_mel_ce: 3.786707878112793 (4.013959572655817) + | > loss: 0.9580538868904114 (1.0142155575184617) + | > current_lr: 5e-06 + | > step_time: 0.3411 (0.3239561114992413) + | > loader_time: 0.0091 (0.011809969402494892) + + + --> TIME: 2024-10-20 04:35:42 -- STEP: 1100/16928 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.04235095530748367 (0.04288607301359829) + | > loss_mel_ce: 4.022974967956543 (4.012209328087894) + | > loss: 1.016331434249878 (1.0137738506360505) + | > current_lr: 5e-06 + | > step_time: 0.3214 (0.32454654476859335) + | > loader_time: 0.0104 (0.011775133826515901) + + + --> TIME: 2024-10-20 04:36:09 -- STEP: 1150/16928 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.04375528171658516 (0.04283648898744067) + | > loss_mel_ce: 3.823617935180664 (4.009655345626496) + | > loss: 0.96684330701828 (1.0131229593442859) + | > current_lr: 5e-06 + | > step_time: 0.2523 (0.3244183503026545) + | > loader_time: 0.0118 (0.011740187354709804) + + + --> TIME: 2024-10-20 04:36:37 -- STEP: 1200/16928 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.04314899444580078 (0.042819455663363135) + | > loss_mel_ce: 4.0334792137146 (4.008158496816949) + | > loss: 1.0191570520401 (1.012744488716127) + | > current_lr: 5e-06 + | > step_time: 0.3724 (0.32479146420955635) + | > loader_time: 0.0093 (0.011684101223945632) + + + --> TIME: 2024-10-20 04:37:04 -- STEP: 1250/16928 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.04129737615585327 (0.04280703998506069) + | > loss_mel_ce: 3.961423873901367 (4.008704612922664) + | > loss: 1.0006803274154663 (1.0128779139041906) + | > current_lr: 5e-06 + | > step_time: 0.4387 (0.32491861381530757) + | > loader_time: 0.0093 (0.011611618423461931) + + + --> TIME: 2024-10-20 04:37:30 -- STEP: 1300/16928 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.051332831382751465 (0.042815999804207895) + | > loss_mel_ce: 4.2081217765808105 (4.0066180183337226) + | > loss: 1.064863681793213 (1.0123585052215143) + | > current_lr: 5e-06 + | > step_time: 0.3743 (0.3239586875988886) + | > loader_time: 0.0116 (0.011540954846602225) + + + --> TIME: 2024-10-20 04:37:57 -- STEP: 1350/16928 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.04875096678733826 (0.04278003808524871) + | > loss_mel_ce: 4.370071887969971 (4.002007890983858) + | > loss: 1.1047056913375854 (1.0111969829047174) + | > current_lr: 5e-06 + | > step_time: 0.5022 (0.3239135818128229) + | > loader_time: 0.0158 (0.01149283779991998) + + + --> TIME: 2024-10-20 04:38:24 -- STEP: 1400/16928 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.04382750019431114 (0.042725010346621246) + | > loss_mel_ce: 3.743358850479126 (3.999307453462046) + | > loss: 0.9467965960502625 (1.0105081165688392) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.3240280548163822) + | > loader_time: 0.0128 (0.011452783005578183) + + + --> TIME: 2024-10-20 04:38:50 -- STEP: 1450/16928 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.04447376728057861 (0.04272684093179373) + | > loss_mel_ce: 3.896899461746216 (3.998016538126705) + | > loss: 0.985343337059021 (1.010185845555932) + | > current_lr: 5e-06 + | > step_time: 0.2918 (0.3233947142239271) + | > loader_time: 0.0107 (0.011410125863963165) + + + --> TIME: 2024-10-20 04:39:16 -- STEP: 1500/16928 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.04443904384970665 (0.042692443547149496) + | > loss_mel_ce: 3.917815923690796 (3.9962806102434696) + | > loss: 0.9905637502670288 (1.0097432641585677) + | > current_lr: 5e-06 + | > step_time: 0.4175 (0.3231007461547848) + | > loader_time: 0.0098 (0.011364382266998296) + + + --> TIME: 2024-10-20 04:39:42 -- STEP: 1550/16928 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.03716912120580673 (0.04266777532715951) + | > loss_mel_ce: 4.003957271575928 (3.9948389710149366) + | > loss: 1.0102815628051758 (1.0093766871575396) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.32226558962175883) + | > loader_time: 0.0099 (0.011306852832917248) + + + --> TIME: 2024-10-20 04:40:09 -- STEP: 1600/16928 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.042920880019664764 (0.042664649547077715) + | > loss_mel_ce: 4.4499993324279785 (3.993167245835057) + | > loss: 1.1232300996780396 (1.008957974612714) + | > current_lr: 5e-06 + | > step_time: 0.21 (0.3221853198111052) + | > loader_time: 0.0095 (0.011277656704187398) + + + --> TIME: 2024-10-20 04:40:35 -- STEP: 1650/16928 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.044669490307569504 (0.042665319045384724) + | > loss_mel_ce: 4.41850471496582 (3.9912765774582284) + | > loss: 1.1157935857772827 (1.0084854749116041) + | > current_lr: 5e-06 + | > step_time: 0.398 (0.32212625286795826) + | > loader_time: 0.0093 (0.011246661850900366) + + + --> TIME: 2024-10-20 04:41:04 -- STEP: 1700/16928 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.03987114131450653 (0.04263826800181585) + | > loss_mel_ce: 3.6664843559265137 (3.9879694078950236) + | > loss: 0.9265888929367065 (1.007651919687496) + | > current_lr: 5e-06 + | > step_time: 0.3453 (0.32285109365687603) + | > loader_time: 0.0092 (0.011213589976815621) + + + --> TIME: 2024-10-20 04:41:30 -- STEP: 1750/16928 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.04145665466785431 (0.04262732969863074) + | > loss_mel_ce: 4.135623931884766 (3.9852819618497493) + | > loss: 1.0442701578140259 (1.006977323566165) + | > current_lr: 5e-06 + | > step_time: 0.5019 (0.322356084142412) + | > loader_time: 0.0137 (0.011174988337925506) + + + --> TIME: 2024-10-20 04:41:57 -- STEP: 1800/16928 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.04008308798074722 (0.04261578826854626) + | > loss_mel_ce: 4.535616397857666 (3.9851648860507463) + | > loss: 1.1439248323440552 (1.0069451692369262) + | > current_lr: 5e-06 + | > step_time: 0.3708 (0.32254585623741105) + | > loader_time: 0.0111 (0.011145272122489086) + + + --> TIME: 2024-10-20 04:42:24 -- STEP: 1850/16928 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.03566616773605347 (0.04260619421262999) + | > loss_mel_ce: 3.752776861190796 (3.9824253550091226) + | > loss: 0.9471107721328735 (1.0062578879498159) + | > current_lr: 5e-06 + | > step_time: 0.2536 (0.3225083454235176) + | > loader_time: 0.01 (0.011123208226384348) + + + --> TIME: 2024-10-20 04:42:51 -- STEP: 1900/16928 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.04371032491326332 (0.04259988522843311) + | > loss_mel_ce: 3.7229795455932617 (3.9817926246241444) + | > loss: 0.9416724443435669 (1.006098128180756) + | > current_lr: 5e-06 + | > step_time: 0.2509 (0.3222902453573122) + | > loader_time: 0.0112 (0.011096773398549937) + + + --> TIME: 2024-10-20 04:43:18 -- STEP: 1950/16928 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.04080095514655113 (0.042589408243313816) + | > loss_mel_ce: 4.0782790184021 (3.9794290795692957) + | > loss: 1.029770016670227 (1.0055046227956446) + | > current_lr: 5e-06 + | > step_time: 0.2572 (0.32218366818550265) + | > loader_time: 0.0086 (0.01106177464509622) + + + --> TIME: 2024-10-20 04:43:43 -- STEP: 2000/16928 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.04730158671736717 (0.042591869991272686) + | > loss_mel_ce: 3.8682336807250977 (3.976467968702309) + | > loss: 0.9788838028907776 (1.0047649606764335) + | > current_lr: 5e-06 + | > step_time: 0.291 (0.3214334272146221) + | > loader_time: 0.0105 (0.011028695583343509) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_2000.pth + + --> TIME: 2024-10-20 04:44:19 -- STEP: 2050/16928 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.04611528664827347 (0.04258014807613884) + | > loss_mel_ce: 3.900569438934326 (3.9733520670634874) + | > loss: 0.9866712093353271 (1.0039830546844315) + | > current_lr: 5e-06 + | > step_time: 0.3791 (0.32082297360024764) + | > loader_time: 0.0105 (0.01101922558575142) + + + --> TIME: 2024-10-20 04:44:47 -- STEP: 2100/16928 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.04400363191962242 (0.042557555348390626) + | > loss_mel_ce: 3.7283310890197754 (3.9721050960676942) + | > loss: 0.9430837035179138 (1.0036656635148202) + | > current_lr: 5e-06 + | > step_time: 0.3041 (0.32109003543853715) + | > loader_time: 0.0088 (0.010987106504894442) + + + --> TIME: 2024-10-20 04:45:14 -- STEP: 2150/16928 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.038768161088228226 (0.042543576607523954) + | > loss_mel_ce: 3.7931225299835205 (3.9699178645777082) + | > loss: 0.9579726457595825 (1.0031153610695256) + | > current_lr: 5e-06 + | > step_time: 0.259 (0.321122090095697) + | > loader_time: 0.009 (0.010956717757291576) + + + --> TIME: 2024-10-20 04:45:41 -- STEP: 2200/16928 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.0410122349858284 (0.042547171051529324) + | > loss_mel_ce: 3.57930064201355 (3.9681762785261263) + | > loss: 0.9050782322883606 (1.0026808632232938) + | > current_lr: 5e-06 + | > step_time: 0.3987 (0.32101883454756275) + | > loader_time: 0.0112 (0.010927559354088524) + + + --> TIME: 2024-10-20 04:46:09 -- STEP: 2250/16928 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.04270458593964577 (0.04252638870643246) + | > loss_mel_ce: 3.8876686096191406 (3.9659291436937054) + | > loss: 0.982593297958374 (1.0021138838397143) + | > current_lr: 5e-06 + | > step_time: 0.3139 (0.32136773120032386) + | > loader_time: 0.011 (0.010893451372782388) + + + --> TIME: 2024-10-20 04:46:36 -- STEP: 2300/16928 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.040422018617391586 (0.04250697064496901) + | > loss_mel_ce: 4.2723236083984375 (3.965003918875812) + | > loss: 1.0781863927841187 (1.0018777231289004) + | > current_lr: 5e-06 + | > step_time: 0.2365 (0.32144306379815757) + | > loader_time: 0.0101 (0.010861589804939597) + + + --> TIME: 2024-10-20 04:47:03 -- STEP: 2350/16928 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.04230131208896637 (0.04248260513106559) + | > loss_mel_ce: 4.080714225769043 (3.962500033378595) + | > loss: 1.0307538509368896 (1.0012456603506792) + | > current_lr: 5e-06 + | > step_time: 0.3587 (0.32146808289466966) + | > loader_time: 0.0089 (0.010825743370867783) + + + --> TIME: 2024-10-20 04:47:29 -- STEP: 2400/16928 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.042303651571273804 (0.042475143953536944) + | > loss_mel_ce: 3.6935677528381348 (3.9622573461135167) + | > loss: 0.9339678287506104 (1.0011831232657056) + | > current_lr: 5e-06 + | > step_time: 0.2459 (0.32126513123512207) + | > loader_time: 0.0086 (0.01079762309789657) + + + --> TIME: 2024-10-20 04:47:57 -- STEP: 2450/16928 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.04272953048348427 (0.042460759269947905) + | > loss_mel_ce: 4.465768814086914 (3.9612476249616915) + | > loss: 1.127124547958374 (1.0009270967512736) + | > current_lr: 5e-06 + | > step_time: 0.3596 (0.3215240484354441) + | > loader_time: 0.0115 (0.010772796650322112) + + + --> TIME: 2024-10-20 04:48:23 -- STEP: 2500/16928 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.03703200817108154 (0.04245310117006302) + | > loss_mel_ce: 3.761725902557373 (3.958552928256983) + | > loss: 0.949689507484436 (1.0002515078783059) + | > current_lr: 5e-06 + | > step_time: 0.2005 (0.3212566002845757) + | > loader_time: 0.0089 (0.010745587253570549) + + + --> TIME: 2024-10-20 04:48:50 -- STEP: 2550/16928 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.05239059031009674 (0.04242424850370367) + | > loss_mel_ce: 3.5093090534210205 (3.956046227286838) + | > loss: 0.890424907207489 (0.999617619350847) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.3210469847099449) + | > loader_time: 0.0086 (0.010714834437650785) + + + --> TIME: 2024-10-20 04:49:16 -- STEP: 2600/16928 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.040786437690258026 (0.042407262126127096) + | > loss_mel_ce: 3.502037525177002 (3.9539664443639553) + | > loss: 0.8857060074806213 (0.9990934271078866) + | > current_lr: 5e-06 + | > step_time: 0.1839 (0.3208560210924877) + | > loader_time: 0.0083 (0.01068453871286832) + + + --> TIME: 2024-10-20 04:49:43 -- STEP: 2650/16928 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.039015963673591614 (0.0423893907919245) + | > loss_mel_ce: 3.8949432373046875 (3.952292887849622) + | > loss: 0.9834898114204407 (0.998670570108128) + | > current_lr: 5e-06 + | > step_time: 0.253 (0.32087170807820414) + | > loader_time: 0.0099 (0.010666695630775302) + + + --> TIME: 2024-10-20 04:50:08 -- STEP: 2700/16928 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.03850698471069336 (0.04238522217367532) + | > loss_mel_ce: 3.906118869781494 (3.949673755610425) + | > loss: 0.9861564636230469 (0.9980147448734022) + | > current_lr: 5e-06 + | > step_time: 0.2703 (0.32049554621731763) + | > loader_time: 0.0088 (0.01063540361545703) + + + --> TIME: 2024-10-20 04:50:37 -- STEP: 2750/16928 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.03726606070995331 (0.04237427518178115) + | > loss_mel_ce: 4.074252128601074 (3.9479468664689445) + | > loss: 1.0278795957565308 (0.9975802858742822) + | > current_lr: 5e-06 + | > step_time: 0.2481 (0.3209007577896116) + | > loader_time: 0.0084 (0.010627307545055037) + + + --> TIME: 2024-10-20 04:51:03 -- STEP: 2800/16928 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.041797250509262085 (0.042355138384071817) + | > loss_mel_ce: 4.091737270355225 (3.9483927450009704) + | > loss: 1.0333836078643799 (0.9976869712770007) + | > current_lr: 5e-06 + | > step_time: 0.2901 (0.32089949156556813) + | > loader_time: 0.0101 (0.010607672844614292) + + + --> TIME: 2024-10-20 04:51:32 -- STEP: 2850/16928 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.04214777424931526 (0.042335581625239906) + | > loss_mel_ce: 3.6356394290924072 (3.9475681816067563) + | > loss: 0.9194468259811401 (0.9974759412230128) + | > current_lr: 5e-06 + | > step_time: 0.1909 (0.3213221089881764) + | > loader_time: 0.01 (0.010599241674992063) + + + --> TIME: 2024-10-20 04:51:58 -- STEP: 2900/16928 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.04028052091598511 (0.042326281597388236) + | > loss_mel_ce: 3.506361246109009 (3.9464855719434753) + | > loss: 0.8866604566574097 (0.9972029637468293) + | > current_lr: 5e-06 + | > step_time: 0.2892 (0.32110059014682135) + | > loader_time: 0.0102 (0.010591378047548476) + + + --> TIME: 2024-10-20 04:52:24 -- STEP: 2950/16928 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.03858231008052826 (0.042313319999030104) + | > loss_mel_ce: 3.6365349292755127 (3.946143356420221) + | > loss: 0.9187793135643005 (0.997114169415783) + | > current_lr: 5e-06 + | > step_time: 0.3117 (0.320897097425946) + | > loader_time: 0.0103 (0.010582627280283767) + + + --> TIME: 2024-10-20 04:52:51 -- STEP: 3000/16928 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.039442937821149826 (0.04229415061076485) + | > loss_mel_ce: 4.49899435043335 (3.9440626991589816) + | > loss: 1.134609341621399 (0.9965892127553642) + | > current_lr: 5e-06 + | > step_time: 0.2472 (0.3207896634737653) + | > loader_time: 0.0094 (0.010570651769638045) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_3000.pth + + --> TIME: 2024-10-20 04:53:33 -- STEP: 3050/16928 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.04170186445116997 (0.04227978396244717) + | > loss_mel_ce: 3.7817301750183105 (3.942165466058445) + | > loss: 0.9558579921722412 (0.9961113127528664) + | > current_lr: 5e-06 + | > step_time: 0.359 (0.320840856051836) + | > loader_time: 0.0102 (0.010567097663879376) + + + --> TIME: 2024-10-20 04:54:00 -- STEP: 3100/16928 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.04687758907675743 (0.04227237937671527) + | > loss_mel_ce: 4.859328269958496 (3.9403104916695577) + | > loss: 1.2265514135360718 (0.9956457180361613) + | > current_lr: 5e-06 + | > step_time: 0.3448 (0.32088503299220944) + | > loader_time: 0.0095 (0.01055663001152775) + + + --> TIME: 2024-10-20 04:54:27 -- STEP: 3150/16928 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.04331110790371895 (0.0422576064651921) + | > loss_mel_ce: 3.403353691101074 (3.939465733558408) + | > loss: 0.861666202545166 (0.995430835277317) + | > current_lr: 5e-06 + | > step_time: 0.2497 (0.32077979049985383) + | > loader_time: 0.0089 (0.010541988554454967) + + + --> TIME: 2024-10-20 04:54:54 -- STEP: 3200/16928 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.03944612666964531 (0.04225783649482768) + | > loss_mel_ce: 3.534641742706299 (3.938585309386249) + | > loss: 0.8935219645500183 (0.9952107867226023) + | > current_lr: 5e-06 + | > step_time: 0.2513 (0.3208354219794275) + | > loader_time: 0.0096 (0.010530398562550527) + + + --> TIME: 2024-10-20 04:55:19 -- STEP: 3250/16928 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.03976767137646675 (0.04224297431340588) + | > loss_mel_ce: 3.8930916786193848 (3.9369519628378016) + | > loss: 0.9832148551940918 (0.9947987344815199) + | > current_lr: 5e-06 + | > step_time: 0.363 (0.3205406931363621) + | > loader_time: 0.0091 (0.010519684057969296) + + + --> TIME: 2024-10-20 04:55:46 -- STEP: 3300/16928 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.04137289524078369 (0.04222792106257247) + | > loss_mel_ce: 4.357752799987793 (3.9359060858957653) + | > loss: 1.0997813940048218 (0.9945335018996054) + | > current_lr: 5e-06 + | > step_time: 0.2095 (0.32051518945982976) + | > loader_time: 0.0104 (0.010511775233528814) + + + --> TIME: 2024-10-20 04:56:13 -- STEP: 3350/16928 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.039716269820928574 (0.042214578093210296) + | > loss_mel_ce: 4.206035137176514 (3.935290066092757) + | > loss: 1.0614378452301025 (0.9943761612052365) + | > current_lr: 5e-06 + | > step_time: 0.3399 (0.3205956553701146) + | > loader_time: 0.0104 (0.010503658536654782) + + + --> TIME: 2024-10-20 04:56:42 -- STEP: 3400/16928 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.04037037864327431 (0.042193611625801146) + | > loss_mel_ce: 3.6392486095428467 (3.9339015204064944) + | > loss: 0.9199047684669495 (0.9940237831718799) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.32084158665993645) + | > loader_time: 0.0088 (0.010494452013688912) + + + --> TIME: 2024-10-20 04:57:07 -- STEP: 3450/16928 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.03822580352425575 (0.042176942356686696) + | > loss_mel_ce: 4.1481451988220215 (3.9316216839223626) + | > loss: 1.0465927124023438 (0.9934496568251364) + | > current_lr: 5e-06 + | > step_time: 0.38 (0.32051429686339017) + | > loader_time: 0.0101 (0.010481056545091695) + + + --> TIME: 2024-10-20 04:57:34 -- STEP: 3500/16928 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.04062354937195778 (0.04215626324926107) + | > loss_mel_ce: 4.030601501464844 (3.930917870589661) + | > loss: 1.0178062915802002 (0.9932685337236966) + | > current_lr: 5e-06 + | > step_time: 0.2368 (0.32056653186253153) + | > loader_time: 0.0131 (0.01047517858232769) + + + --> TIME: 2024-10-20 04:58:00 -- STEP: 3550/16928 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.04613318294286728 (0.042143048760546795) + | > loss_mel_ce: 3.6658010482788086 (3.928997707165459) + | > loss: 0.9279835820198059 (0.9927851892357158) + | > current_lr: 5e-06 + | > step_time: 0.2728 (0.32040770658305007) + | > loader_time: 0.0099 (0.010466219337893188) + + + --> TIME: 2024-10-20 04:58:28 -- STEP: 3600/16928 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.040802471339702606 (0.04212881911235554) + | > loss_mel_ce: 3.9085404872894287 (3.9274928356541485) + | > loss: 0.987335741519928 (0.9924054139686972) + | > current_lr: 5e-06 + | > step_time: 0.3342 (0.3205865455998316) + | > loader_time: 0.0113 (0.01045705894629159) + + + --> TIME: 2024-10-20 04:58:54 -- STEP: 3650/16928 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.042915768921375275 (0.0421149210346072) + | > loss_mel_ce: 3.7664828300476074 (3.9257827895961355) + | > loss: 0.9523496627807617 (0.9919744278796748) + | > current_lr: 5e-06 + | > step_time: 0.2767 (0.32052900366587195) + | > loader_time: 0.0103 (0.010453855109541369) + + + --> TIME: 2024-10-20 04:59:20 -- STEP: 3700/16928 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.040143661201000214 (0.042100765473536576) + | > loss_mel_ce: 4.299578666687012 (3.9250565945779914) + | > loss: 1.0849305391311646 (0.9917893401513245) + | > current_lr: 5e-06 + | > step_time: 0.2716 (0.32026870547114206) + | > loader_time: 0.0101 (0.01044647055703239) + + + --> TIME: 2024-10-20 04:59:48 -- STEP: 3750/16928 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.04047971963882446 (0.04209118569095933) + | > loss_mel_ce: 3.5082876682281494 (3.9238058866500816) + | > loss: 0.8871918320655823 (0.9914742681980149) + | > current_lr: 5e-06 + | > step_time: 0.1906 (0.32050603790283216) + | > loader_time: 0.0109 (0.010439526430765773) + + + --> TIME: 2024-10-20 05:00:14 -- STEP: 3800/16928 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.041252560913562775 (0.04207797626719666) + | > loss_mel_ce: 3.60019850730896 (3.9222528305806574) + | > loss: 0.9103627800941467 (0.9910827018242149) + | > current_lr: 5e-06 + | > step_time: 0.2093 (0.32042121065290363) + | > loader_time: 0.0088 (0.010430955510390418) + + + --> TIME: 2024-10-20 05:00:40 -- STEP: 3850/16928 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.0404001846909523 (0.042062877335331685) + | > loss_mel_ce: 3.597712278366089 (3.92115962109008) + | > loss: 0.9095281362533569 (0.9908056248318081) + | > current_lr: 5e-06 + | > step_time: 0.2686 (0.32036504045709396) + | > loader_time: 0.0096 (0.010420688653921137) + + + --> TIME: 2024-10-20 05:01:08 -- STEP: 3900/16928 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.03993711248040199 (0.042052464343798494) + | > loss_mel_ce: 3.9744701385498047 (3.9200343592350264) + | > loss: 1.0036017894744873 (0.9905217061592998) + | > current_lr: 5e-06 + | > step_time: 0.2533 (0.3204885001671621) + | > loader_time: 0.0084 (0.010406993535848749) + + + --> TIME: 2024-10-20 05:01:35 -- STEP: 3950/16928 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.04463174566626549 (0.0420481251321639) + | > loss_mel_ce: 3.5714938640594482 (3.9186021404628475) + | > loss: 0.9040313959121704 (0.9901625666437285) + | > current_lr: 5e-06 + | > step_time: 0.2518 (0.32056983271731615) + | > loader_time: 0.009 (0.010395997385435451) + + + --> TIME: 2024-10-20 05:02:03 -- STEP: 4000/16928 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.03988657891750336 (0.042038818025030227) + | > loss_mel_ce: 3.7371463775634766 (3.917495527803894) + | > loss: 0.9442582130432129 (0.9898835867196336) + | > current_lr: 5e-06 + | > step_time: 0.3706 (0.3205992397665979) + | > loader_time: 0.0092 (0.010385426163673387) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_4000.pth + + --> TIME: 2024-10-20 05:02:43 -- STEP: 4050/16928 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.04573066532611847 (0.04202710009835387) + | > loss_mel_ce: 4.285260200500488 (3.916976509035366) + | > loss: 1.0827476978302002 (0.9897509024908527) + | > current_lr: 5e-06 + | > step_time: 0.2574 (0.3204685660939159) + | > loader_time: 0.0095 (0.010383500464168582) + + + --> TIME: 2024-10-20 05:03:08 -- STEP: 4100/16928 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.04248099401593208 (0.04201457731698347) + | > loss_mel_ce: 3.8191864490509033 (3.915830238156199) + | > loss: 0.9654168486595154 (0.9894612040461578) + | > current_lr: 5e-06 + | > step_time: 0.2497 (0.32032128613169614) + | > loader_time: 0.0096 (0.010367540731662643) + + + --> TIME: 2024-10-20 05:03:34 -- STEP: 4150/16928 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.04247216507792473 (0.042005446397935076) + | > loss_mel_ce: 3.6857151985168457 (3.915273163806958) + | > loss: 0.9320468306541443 (0.9893196527211076) + | > current_lr: 5e-06 + | > step_time: 0.4405 (0.320209082005972) + | > loader_time: 0.0089 (0.010351838318698361) + + + --> TIME: 2024-10-20 05:04:02 -- STEP: 4200/16928 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.040488351136446 (0.04200554596437587) + | > loss_mel_ce: 3.246023416519165 (3.9147471434729404) + | > loss: 0.8216279149055481 (0.9891881724766336) + | > current_lr: 5e-06 + | > step_time: 0.1543 (0.3203672665641424) + | > loader_time: 0.0088 (0.010336628527868347) + + + --> TIME: 2024-10-20 05:04:28 -- STEP: 4250/16928 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.03767656162381172 (0.041986747777637334) + | > loss_mel_ce: 3.7397079467773438 (3.913703026547148) + | > loss: 0.9443461298942566 (0.9889224437405094) + | > current_lr: 5e-06 + | > step_time: 0.3363 (0.3201712640313544) + | > loader_time: 0.0087 (0.01032540573793298) + + + --> TIME: 2024-10-20 05:04:54 -- STEP: 4300/16928 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.04510113224387169 (0.04197169707265014) + | > loss_mel_ce: 3.8094444274902344 (3.9128916516969334) + | > loss: 0.9636363983154297 (0.9887158373760635) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.3200943098511809) + | > loader_time: 0.0083 (0.010314636784930551) + + + --> TIME: 2024-10-20 05:05:20 -- STEP: 4350/16928 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.03930208832025528 (0.0419677067094151) + | > loss_mel_ce: 4.79104471206665 (3.9125716039504095) + | > loss: 1.207586646080017 (0.9886348277947011) + | > current_lr: 5e-06 + | > step_time: 0.2503 (0.3199846591620612) + | > loader_time: 0.0087 (0.010307765281063377) + + + --> TIME: 2024-10-20 05:05:47 -- STEP: 4400/16928 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.04199834167957306 (0.04195583465678453) + | > loss_mel_ce: 3.969235420227051 (3.911442516066808) + | > loss: 1.0028084516525269 (0.9883495877683175) + | > current_lr: 5e-06 + | > step_time: 0.2873 (0.31991630283269035) + | > loader_time: 0.0087 (0.010299064191904928) + + + --> TIME: 2024-10-20 05:06:15 -- STEP: 4450/16928 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.043098434805870056 (0.041952638731578805) + | > loss_mel_ce: 4.104319095611572 (3.911059667876594) + | > loss: 1.0368543863296509 (0.9882530767864068) + | > current_lr: 5e-06 + | > step_time: 0.4619 (0.32007939033294014) + | > loader_time: 0.0096 (0.010291877703720264) + + + --> TIME: 2024-10-20 05:06:41 -- STEP: 4500/16928 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.040037382394075394 (0.04193927627222408) + | > loss_mel_ce: 4.075897693634033 (3.9102826801935797) + | > loss: 1.02898371219635 (0.988055489142737) + | > current_lr: 5e-06 + | > step_time: 0.2741 (0.3199958942201405) + | > loader_time: 0.0092 (0.010281123002370194) + + + --> TIME: 2024-10-20 05:07:07 -- STEP: 4550/16928 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.03925936296582222 (0.04192546223546126) + | > loss_mel_ce: 4.088086128234863 (3.9093224583091284) + | > loss: 1.0318363904953003 (0.9878119801426993) + | > current_lr: 5e-06 + | > step_time: 0.5074 (0.31989565844064266) + | > loader_time: 0.0099 (0.010270450351002451) + + + --> TIME: 2024-10-20 05:07:34 -- STEP: 4600/16928 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.04078835994005203 (0.0419094414069601) + | > loss_mel_ce: 3.6473190784454346 (3.9085849981722594) + | > loss: 0.9220268726348877 (0.9876236099004757) + | > current_lr: 5e-06 + | > step_time: 0.2293 (0.319938047087711) + | > loader_time: 0.0097 (0.010258374473322987) + + + --> TIME: 2024-10-20 05:08:01 -- STEP: 4650/16928 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.04150089994072914 (0.041899136102167535) + | > loss_mel_ce: 3.792029619216919 (3.9071915950570024) + | > loss: 0.9583826065063477 (0.9872726827411253) + | > current_lr: 5e-06 + | > step_time: 0.4 (0.31996802150562254) + | > loader_time: 0.0087 (0.010247316360473628) + + + --> TIME: 2024-10-20 05:08:27 -- STEP: 4700/16928 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.03902335837483406 (0.04189271208136641) + | > loss_mel_ce: 3.8589084148406982 (3.906167855820754) + | > loss: 0.9744829535484314 (0.9870151419462053) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.3197733947571288) + | > loader_time: 0.0099 (0.010235131294169322) + + + --> TIME: 2024-10-20 05:08:54 -- STEP: 4750/16928 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.03811630234122276 (0.04188463149337394) + | > loss_mel_ce: 3.583906888961792 (3.9055002151790386) + | > loss: 0.9055057764053345 (0.9868462116467337) + | > current_lr: 5e-06 + | > step_time: 0.2546 (0.31982218099895265) + | > loader_time: 0.0083 (0.010223329594260769) + + + --> TIME: 2024-10-20 05:09:20 -- STEP: 4800/16928 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.0397045761346817 (0.041870172809964676) + | > loss_mel_ce: 3.4541964530944824 (3.9048355018099117) + | > loss: 0.8734752535820007 (0.9866764185825995) + | > current_lr: 5e-06 + | > step_time: 0.3055 (0.3197982956965762) + | > loader_time: 0.0093 (0.010211884677410132) + + + --> TIME: 2024-10-20 05:09:48 -- STEP: 4850/16928 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.036917269229888916 (0.04185916323099556) + | > loss_mel_ce: 4.359528064727783 (3.9047831477823918) + | > loss: 1.0991113185882568 (0.9866605777470118) + | > current_lr: 5e-06 + | > step_time: 0.311 (0.3199263021134836) + | > loader_time: 0.009 (0.01020309222113226) + + + --> TIME: 2024-10-20 05:10:13 -- STEP: 4900/16928 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.041162408888339996 (0.041846179313653586) + | > loss_mel_ce: 4.179596424102783 (3.9039232795092493) + | > loss: 1.0551897287368774 (0.9864423646732263) + | > current_lr: 5e-06 + | > step_time: 0.2887 (0.31972840557293014) + | > loader_time: 0.0097 (0.010191249068902468) + + + --> TIME: 2024-10-20 05:10:42 -- STEP: 4950/16928 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.03736382722854614 (0.04183575968549707) + | > loss_mel_ce: 4.143190383911133 (3.9030185805426676) + | > loss: 1.0451385974884033 (0.986213585102198) + | > current_lr: 5e-06 + | > step_time: 0.444 (0.3199199343209317) + | > loader_time: 0.011 (0.010181827256173804) + + + --> TIME: 2024-10-20 05:11:10 -- STEP: 5000/16928 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.041073668748140335 (0.04182385997995731) + | > loss_mel_ce: 3.693619966506958 (3.9025444356441468) + | > loss: 0.9336733818054199 (0.9860920740365993) + | > current_lr: 5e-06 + | > step_time: 0.2199 (0.32007803692817693) + | > loader_time: 0.0084 (0.010172494935989384) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_5000.pth + + --> TIME: 2024-10-20 05:11:46 -- STEP: 5050/16928 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.04245426505804062 (0.04181649803746451) + | > loss_mel_ce: 3.6687510013580322 (3.9012243498906023) + | > loss: 0.9278013110160828 (0.985760212086216) + | > current_lr: 5e-06 + | > step_time: 0.3579 (0.3198177525076536) + | > loader_time: 0.01 (0.01016669896569583) + + + --> TIME: 2024-10-20 05:12:12 -- STEP: 5100/16928 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.044096291065216064 (0.04180349708406943) + | > loss_mel_ce: 3.7685930728912354 (3.9000688004493687) + | > loss: 0.9531723260879517 (0.9854680744806936) + | > current_lr: 5e-06 + | > step_time: 0.3012 (0.31972627429401174) + | > loader_time: 0.009 (0.010157912291732492) + + + --> TIME: 2024-10-20 05:12:40 -- STEP: 5150/16928 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.040284767746925354 (0.04179380047162174) + | > loss_mel_ce: 3.925039291381836 (3.8990885953532812) + | > loss: 0.9913310408592224 (0.985220599035616) + | > current_lr: 5e-06 + | > step_time: 0.3061 (0.3198775490973759) + | > loader_time: 0.0082 (0.010150027784329014) + + + --> TIME: 2024-10-20 05:13:06 -- STEP: 5200/16928 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.041931699961423874 (0.041791732339188405) + | > loss_mel_ce: 3.8596200942993164 (3.8978533882361166) + | > loss: 0.9753879308700562 (0.9849112801941552) + | > current_lr: 5e-06 + | > step_time: 0.2727 (0.31984630593886754) + | > loader_time: 0.01 (0.010141881245833175) + + + --> TIME: 2024-10-20 05:13:33 -- STEP: 5250/16928 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.042557328939437866 (0.04178411371793061) + | > loss_mel_ce: 2.9677023887634277 (3.896565841220671) + | > loss: 0.7525649070739746 (0.9845874887648093) + | > current_lr: 5e-06 + | > step_time: 0.191 (0.3199034092766901) + | > loader_time: 0.009 (0.01013485163734072) + + + --> TIME: 2024-10-20 05:14:00 -- STEP: 5300/16928 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.03827071562409401 (0.04177516382468752) + | > loss_mel_ce: 3.9816184043884277 (3.8956602126247453) + | > loss: 1.0049723386764526 (0.9843588441610347) + | > current_lr: 5e-06 + | > step_time: 0.2873 (0.3198812975973455) + | > loader_time: 0.0091 (0.010128518410448753) + + + --> TIME: 2024-10-20 05:14:27 -- STEP: 5350/16928 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.04095086082816124 (0.04176266019867954) + | > loss_mel_ce: 3.9502933025360107 (3.8949644636885) + | > loss: 0.9978110194206238 (0.9841817810156646) + | > current_lr: 5e-06 + | > step_time: 0.3402 (0.3199325090033987) + | > loader_time: 0.0092 (0.010124235777097325) + + + --> TIME: 2024-10-20 05:14:55 -- STEP: 5400/16928 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.03609229251742363 (0.04175350038196748) + | > loss_mel_ce: 3.55694580078125 (3.8938946949552578) + | > loss: 0.8982595205307007 (0.9839120489027775) + | > current_lr: 5e-06 + | > step_time: 0.2316 (0.319989656430704) + | > loader_time: 0.0097 (0.010119814828590102) + + + --> TIME: 2024-10-20 05:15:21 -- STEP: 5450/16928 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.03834477439522743 (0.04174346283283249) + | > loss_mel_ce: 4.192458152770996 (3.893074416020592) + | > loss: 1.057700753211975 (0.9837044697945281) + | > current_lr: 5e-06 + | > step_time: 0.4607 (0.3200166506723529) + | > loader_time: 0.0092 (0.010110948720109567) + + + --> TIME: 2024-10-20 05:15:49 -- STEP: 5500/16928 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.03881116211414337 (0.04173600654507218) + | > loss_mel_ce: 4.41536283493042 (3.8925143287398574) + | > loss: 1.1135435104370117 (0.9835625839558525) + | > current_lr: 5e-06 + | > step_time: 0.2734 (0.32013518645546685) + | > loader_time: 0.0095 (0.010104650237343516) + + + --> TIME: 2024-10-20 05:16:17 -- STEP: 5550/16928 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.041524969041347504 (0.0417231918965373) + | > loss_mel_ce: 3.463413715362549 (3.891111412606795) + | > loss: 0.8762346506118774 (0.9832086512956543) + | > current_lr: 5e-06 + | > step_time: 0.3062 (0.32022718004278294) + | > loader_time: 0.0108 (0.010099852665050603) + + + --> TIME: 2024-10-20 05:16:42 -- STEP: 5600/16928 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.04663239046931267 (0.041713862346618225) + | > loss_mel_ce: 4.074362277984619 (3.8902559650370026) + | > loss: 1.0302486419677734 (0.982992457013046) + | > current_lr: 5e-06 + | > step_time: 0.2521 (0.3201036594169488) + | > loader_time: 0.0082 (0.01009199538401194) + + + --> TIME: 2024-10-20 05:17:11 -- STEP: 5650/16928 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.04023416340351105 (0.04169704154661262) + | > loss_mel_ce: 3.6182072162628174 (3.8893205769716084) + | > loss: 0.9146103262901306 (0.9827544047980191) + | > current_lr: 5e-06 + | > step_time: 0.2476 (0.32035679762342406) + | > loader_time: 0.0085 (0.010086406859676378) + + + --> TIME: 2024-10-20 05:17:38 -- STEP: 5700/16928 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.041138019412755966 (0.041694461386323926) + | > loss_mel_ce: 3.528581380844116 (3.8878327855310917) + | > loss: 0.8924298286437988 (0.9823818119157834) + | > current_lr: 5e-06 + | > step_time: 0.3453 (0.32029289747539363) + | > loader_time: 0.0103 (0.010081725162372237) + + + --> TIME: 2024-10-20 05:18:05 -- STEP: 5750/16928 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.039566636085510254 (0.041684797433407285) + | > loss_mel_ce: 4.302434921264648 (3.8880298342497426) + | > loss: 1.0855003595352173 (0.9824286580811386) + | > current_lr: 5e-06 + | > step_time: 0.3042 (0.32032178173894527) + | > loader_time: 0.0093 (0.010077338882114553) + + + --> TIME: 2024-10-20 05:18:31 -- STEP: 5800/16928 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.038629401475191116 (0.04167023231949772) + | > loss_mel_ce: 3.8986761569976807 (3.8869143955049816) + | > loss: 0.9843263626098633 (0.9821461571187818) + | > current_lr: 5e-06 + | > step_time: 0.2709 (0.32021092797147904) + | > loader_time: 0.0101 (0.01007250243219834) + + + --> TIME: 2024-10-20 05:18:58 -- STEP: 5850/16928 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.035436227917671204 (0.041657857200146586) + | > loss_mel_ce: 3.795816421508789 (3.8864391530273283) + | > loss: 0.9578131437301636 (0.9820242526898026) + | > current_lr: 5e-06 + | > step_time: 0.2878 (0.320264536091406) + | > loader_time: 0.0102 (0.010065038183815442) + + + --> TIME: 2024-10-20 05:19:26 -- STEP: 5900/16928 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.04098930209875107 (0.04164384471662958) + | > loss_mel_ce: 3.6019632816314697 (3.8857785305330284) + | > loss: 0.9107381701469421 (0.9818555939803698) + | > current_lr: 5e-06 + | > step_time: 0.1707 (0.32033469305200174) + | > loader_time: 0.0106 (0.010058529740672976) + + + --> TIME: 2024-10-20 05:19:52 -- STEP: 5950/16928 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.041275110095739365 (0.041631974316194094) + | > loss_mel_ce: 3.863009452819824 (3.8847033051482747) + | > loss: 0.9760711193084717 (0.9815838200705401) + | > current_lr: 5e-06 + | > step_time: 0.3587 (0.32030101852256765) + | > loader_time: 0.0091 (0.010050929734686815) + + + --> TIME: 2024-10-20 05:20:19 -- STEP: 6000/16928 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.0398016981780529 (0.04162118286080641) + | > loss_mel_ce: 3.5538463592529297 (3.883724995851514) + | > loss: 0.8984119892120361 (0.9813365448713312) + | > current_lr: 5e-06 + | > step_time: 0.2494 (0.3202865096330647) + | > loader_time: 0.0086 (0.010044975916544569) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_6000.pth + + --> TIME: 2024-10-20 05:20:57 -- STEP: 6050/16928 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.043533723801374435 (0.041613391171063094) + | > loss_mel_ce: 3.9725570678710938 (3.8833287227646354) + | > loss: 1.0040227174758911 (0.9812355286621859) + | > current_lr: 5e-06 + | > step_time: 0.334 (0.32025164824872016) + | > loader_time: 0.0093 (0.01004265647289179) + + + --> TIME: 2024-10-20 05:21:25 -- STEP: 6100/16928 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.041406478732824326 (0.041601903152758773) + | > loss_mel_ce: 3.9076321125030518 (3.8823851500182833) + | > loss: 0.9872596263885498 (0.9809967634619269) + | > current_lr: 5e-06 + | > step_time: 0.3232 (0.3203565957507152) + | > loader_time: 0.0095 (0.010036152386274469) + + + --> TIME: 2024-10-20 05:21:51 -- STEP: 6150/16928 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.03866482898592949 (0.04159194801457026) + | > loss_mel_ce: 4.293724060058594 (3.881550964417494) + | > loss: 1.083097219467163 (0.9807857282762614) + | > current_lr: 5e-06 + | > step_time: 0.2548 (0.32032512773343236) + | > loader_time: 0.0089 (0.01002846109188667) + + + --> TIME: 2024-10-20 05:22:18 -- STEP: 6200/16928 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.03609535098075867 (0.04158754965831177) + | > loss_mel_ce: 3.756418466567993 (3.881149272995608) + | > loss: 0.9481284618377686 (0.9806842058416345) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.32033097309450953) + | > loader_time: 0.0104 (0.010021391453281498) + + + --> TIME: 2024-10-20 05:22:44 -- STEP: 6250/16928 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.041199468076229095 (0.04157316322982293) + | > loss_mel_ce: 4.053049564361572 (3.880831192626951) + | > loss: 1.0235623121261597 (0.9806010891723641) + | > current_lr: 5e-06 + | > step_time: 0.3092 (0.32022888065338145) + | > loader_time: 0.0081 (0.010015056571960418) + + + --> TIME: 2024-10-20 05:23:12 -- STEP: 6300/16928 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.03981949761509895 (0.04156593123480423) + | > loss_mel_ce: 3.406575918197632 (3.8798125173932005) + | > loss: 0.8615988492965698 (0.9803446123713547) + | > current_lr: 5e-06 + | > step_time: 0.1874 (0.32034823349543956) + | > loader_time: 0.0091 (0.010009585259452669) + + + --> TIME: 2024-10-20 05:23:39 -- STEP: 6350/16928 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.039642367511987686 (0.04155703168861022) + | > loss_mel_ce: 4.1771464347839355 (3.8794817490840496) + | > loss: 1.0541971921920776 (0.9802596953910175) + | > current_lr: 5e-06 + | > step_time: 0.248 (0.32037532896507404) + | > loader_time: 0.0083 (0.010003758903563469) + + + --> TIME: 2024-10-20 05:24:06 -- STEP: 6400/16928 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.040486495941877365 (0.04154435551841726) + | > loss_mel_ce: 3.6548655033111572 (3.87929876796901) + | > loss: 0.9238380193710327 (0.9802107810787866) + | > current_lr: 5e-06 + | > step_time: 0.2093 (0.32037087801843817) + | > loader_time: 0.0089 (0.009996998868882629) + + + --> TIME: 2024-10-20 05:24:32 -- STEP: 6450/16928 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.04293525964021683 (0.041532377587385705) + | > loss_mel_ce: 4.046558380126953 (3.8789553928005582) + | > loss: 1.0223734378814697 (0.9801219428232483) + | > current_lr: 5e-06 + | > step_time: 0.496 (0.3203794417270388) + | > loader_time: 0.0087 (0.009991465871648245) + + + --> TIME: 2024-10-20 05:25:01 -- STEP: 6500/16928 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.03809861093759537 (0.04152008576404572) + | > loss_mel_ce: 4.1355743408203125 (3.878251493637376) + | > loss: 1.043418288230896 (0.9799428951006678) + | > current_lr: 5e-06 + | > step_time: 0.2757 (0.320546525551722) + | > loader_time: 0.0098 (0.009988053431877697) + + + --> TIME: 2024-10-20 05:25:27 -- STEP: 6550/16928 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.04430175572633743 (0.04151325371429193) + | > loss_mel_ce: 4.001163482666016 (3.878300847534003) + | > loss: 1.011366367340088 (0.9799535255759735) + | > current_lr: 5e-06 + | > step_time: 0.1898 (0.32047148861047814) + | > loader_time: 0.0091 (0.009982187620556056) + + + --> TIME: 2024-10-20 05:25:54 -- STEP: 6600/16928 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.038937412202358246 (0.04149872870099798) + | > loss_mel_ce: 4.082315921783447 (3.87758945692669) + | > loss: 1.0303133726119995 (0.9797720466779948) + | > current_lr: 5e-06 + | > step_time: 0.373 (0.32044171676491184) + | > loader_time: 0.0091 (0.00997468319806183) + + + --> TIME: 2024-10-20 05:26:20 -- STEP: 6650/16928 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.037007492035627365 (0.04149057199072104) + | > loss_mel_ce: 4.087204933166504 (3.8767382256428977) + | > loss: 1.031053066253662 (0.9795571996843017) + | > current_lr: 5e-06 + | > step_time: 0.361 (0.32038132409404035) + | > loader_time: 0.0093 (0.009969499774445242) + + + --> TIME: 2024-10-20 05:26:46 -- STEP: 6700/16928 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.04429653659462929 (0.041489886511395055) + | > loss_mel_ce: 3.4356794357299805 (3.8771326554355316) + | > loss: 0.8699939846992493 (0.9796556357572336) + | > current_lr: 5e-06 + | > step_time: 0.2077 (0.3203387080377601) + | > loader_time: 0.009 (0.009964786429903375) + + + --> TIME: 2024-10-20 05:27:14 -- STEP: 6750/16928 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.041013654321432114 (0.04148196192968758) + | > loss_mel_ce: 3.741971731185913 (3.8761680527793017) + | > loss: 0.9457463622093201 (0.9794125040019008) + | > current_lr: 5e-06 + | > step_time: 0.3574 (0.3204768617418071) + | > loader_time: 0.0103 (0.009959298946239304) + + + --> TIME: 2024-10-20 05:27:39 -- STEP: 6800/16928 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.03993629664182663 (0.04147430332870589) + | > loss_mel_ce: 3.721674680709839 (3.8752640064674244) + | > loss: 0.9404027462005615 (0.9791845777718468) + | > current_lr: 5e-06 + | > step_time: 0.3206 (0.3202976895079887) + | > loader_time: 0.0094 (0.00995257742264689) + + + --> TIME: 2024-10-20 05:28:06 -- STEP: 6850/16928 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.037874531000852585 (0.041464261967878974) + | > loss_mel_ce: 3.903225898742676 (3.8748142438735385) + | > loss: 0.9852750897407532 (0.9790696267925048) + | > current_lr: 5e-06 + | > step_time: 0.2506 (0.32027839785944745) + | > loader_time: 0.0097 (0.00994686164995177) + + + --> TIME: 2024-10-20 05:28:33 -- STEP: 6900/16928 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.03939463943243027 (0.04145673388750208) + | > loss_mel_ce: 3.8115217685699463 (3.8742237629752205) + | > loss: 0.9627290964126587 (0.9789201245878062) + | > current_lr: 5e-06 + | > step_time: 0.306 (0.3202456808781272) + | > loader_time: 0.0092 (0.009939744541610472) + + + --> TIME: 2024-10-20 05:29:00 -- STEP: 6950/16928 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.03923090547323227 (0.04144516586453349) + | > loss_mel_ce: 3.7178595066070557 (3.872981438602473) + | > loss: 0.9392725825309753 (0.9786066515154126) + | > current_lr: 5e-06 + | > step_time: 0.2261 (0.3202565809977134) + | > loader_time: 0.0096 (0.009935079890189387) + + + --> TIME: 2024-10-20 05:29:26 -- STEP: 7000/16928 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.045489974319934845 (0.04143850417648027) + | > loss_mel_ce: 3.780085802078247 (3.8720987708227956) + | > loss: 0.9563939571380615 (0.9783843191436367) + | > current_lr: 5e-06 + | > step_time: 0.2502 (0.3202373219558165) + | > loader_time: 0.0084 (0.009930256468909103) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_7000.pth + + --> TIME: 2024-10-20 05:30:05 -- STEP: 7050/16928 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.03958956524729729 (0.0414290947356121) + | > loss_mel_ce: 4.2191548347473145 (3.871601467538385) + | > loss: 1.0646860599517822 (0.9782576409468422) + | > current_lr: 5e-06 + | > step_time: 0.4625 (0.3203894532995015) + | > loader_time: 0.01 (0.009929204223849224) + + + --> TIME: 2024-10-20 05:30:32 -- STEP: 7100/16928 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.03969497233629227 (0.04141814319705443) + | > loss_mel_ce: 3.9123432636260986 (3.8712384971094784) + | > loss: 0.9880095720291138 (0.9781641604111236) + | > current_lr: 5e-06 + | > step_time: 0.3547 (0.32042615155099086) + | > loader_time: 0.0097 (0.00992493391036985) + + + --> TIME: 2024-10-20 05:30:59 -- STEP: 7150/16928 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.03837176412343979 (0.041406335209632925) + | > loss_mel_ce: 3.803600788116455 (3.8705403029835286) + | > loss: 0.9604931473731995 (0.9779866598619456) + | > current_lr: 5e-06 + | > step_time: 0.2266 (0.32044387750692244) + | > loader_time: 0.0085 (0.009919348363276105) + + + --> TIME: 2024-10-20 05:31:26 -- STEP: 7200/16928 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.03897126764059067 (0.04139367019188471) + | > loss_mel_ce: 3.5526177883148193 (3.8704426765110735) + | > loss: 0.8978972434997559 (0.9779590869694956) + | > current_lr: 5e-06 + | > step_time: 0.1355 (0.3203764641947211) + | > loader_time: 0.0087 (0.009913181430763645) + + + --> TIME: 2024-10-20 05:31:50 -- STEP: 7250/16928 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.03689339756965637 (0.041387136114054555) + | > loss_mel_ce: 3.701113700866699 (3.869604327267612) + | > loss: 0.9345017671585083 (0.9777478661537178) + | > current_lr: 5e-06 + | > step_time: 0.2561 (0.32021117706956476) + | > loader_time: 0.0088 (0.009906967623480412) + + + --> TIME: 2024-10-20 05:32:16 -- STEP: 7300/16928 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.04169881343841553 (0.04137483351544959) + | > loss_mel_ce: 3.6307668685913086 (3.8688098663826493) + | > loss: 0.9181164503097534 (0.9775461753263873) + | > current_lr: 5e-06 + | > step_time: 0.2876 (0.3201237057659718) + | > loader_time: 0.0088 (0.009901143720705191) + + + --> TIME: 2024-10-20 05:32:43 -- STEP: 7350/16928 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.043807629495859146 (0.041367356417839875) + | > loss_mel_ce: 3.493828773498535 (3.8681689199460587) + | > loss: 0.8844091296195984 (0.9773840694038243) + | > current_lr: 5e-06 + | > step_time: 0.2281 (0.32013698889284664) + | > loader_time: 0.0087 (0.009897175646152603) + + + --> TIME: 2024-10-20 05:33:09 -- STEP: 7400/16928 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.04157492518424988 (0.041356730911175796) + | > loss_mel_ce: 3.362732172012329 (3.8677995207825204) + | > loss: 0.8510767817497253 (0.977289063253919) + | > current_lr: 5e-06 + | > step_time: 0.1832 (0.32004868549269544) + | > loader_time: 0.0093 (0.009892806910179734) + + + --> TIME: 2024-10-20 05:33:36 -- STEP: 7450/16928 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.043727848678827286 (0.04135072688328324) + | > loss_mel_ce: 3.7771239280700684 (3.867220173202103) + | > loss: 0.9552129507064819 (0.9771427253428729) + | > current_lr: 5e-06 + | > step_time: 0.3952 (0.3200484583361831) + | > loader_time: 0.0092 (0.009888985173014163) + + + --> TIME: 2024-10-20 05:34:03 -- STEP: 7500/16928 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.037722647190093994 (0.04134435485998774) + | > loss_mel_ce: 3.73526668548584 (3.866878160794574) + | > loss: 0.9432473182678223 (0.9770556292692828) + | > current_lr: 5e-06 + | > step_time: 0.1737 (0.32008632198969467) + | > loader_time: 0.0091 (0.009886773395538307) + + + --> TIME: 2024-10-20 05:34:29 -- STEP: 7550/16928 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.03997564688324928 (0.04133549918500778) + | > loss_mel_ce: 3.483639717102051 (3.8660362733752502) + | > loss: 0.8809038400650024 (0.9768429434931049) + | > current_lr: 5e-06 + | > step_time: 0.3068 (0.3200347778181361) + | > loader_time: 0.0091 (0.009880570165368873) + + + --> TIME: 2024-10-20 05:34:58 -- STEP: 7600/16928 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.03609522432088852 (0.041320950858490885) + | > loss_mel_ce: 3.3072965145111084 (3.8652126752075375) + | > loss: 0.8358479142189026 (0.9766334068853612) + | > current_lr: 5e-06 + | > step_time: 0.2872 (0.32016197577903094) + | > loader_time: 0.0095 (0.009876793591599693) + + + --> TIME: 2024-10-20 05:35:24 -- STEP: 7650/16928 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.037882331758737564 (0.041310880705227114) + | > loss_mel_ce: 3.1233956813812256 (3.864760851610718) + | > loss: 0.7903195023536682 (0.9765179334902303) + | > current_lr: 5e-06 + | > step_time: 0.3989 (0.3201484490999204) + | > loader_time: 0.0101 (0.009870963532940213) + + + --> TIME: 2024-10-20 05:35:51 -- STEP: 7700/16928 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.04366515576839447 (0.04130619301617905) + | > loss_mel_ce: 3.913487434387207 (3.864355600685266) + | > loss: 0.9892881512641907 (0.976415448839015) + | > current_lr: 5e-06 + | > step_time: 0.3744 (0.32009396683085994) + | > loader_time: 0.0088 (0.009866514763274728) + + + --> TIME: 2024-10-20 05:36:18 -- STEP: 7750/16928 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.038940537720918655 (0.041294050315695274) + | > loss_mel_ce: 3.6173746585845947 (3.863642046713058) + | > loss: 0.9140787720680237 (0.976234024670817) + | > current_lr: 5e-06 + | > step_time: 0.357 (0.3201679845625349) + | > loader_time: 0.0084 (0.009862818010391705) + + + --> TIME: 2024-10-20 05:36:45 -- STEP: 7800/16928 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.04120559245347977 (0.04128050803326247) + | > loss_mel_ce: 4.236025810241699 (3.862730263807833) + | > loss: 1.069307804107666 (0.9760026933214611) + | > current_lr: 5e-06 + | > step_time: 0.2532 (0.32018135153330235) + | > loader_time: 0.0088 (0.009857149888307598) + + + --> TIME: 2024-10-20 05:37:12 -- STEP: 7850/16928 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.036745619028806686 (0.041267725774057205) + | > loss_mel_ce: 4.213570594787598 (3.8621993032078814) + | > loss: 1.0625790357589722 (0.9758667576085237) + | > current_lr: 5e-06 + | > step_time: 0.3989 (0.3201801392379075) + | > loader_time: 0.0091 (0.009851447427348704) + + + --> TIME: 2024-10-20 05:37:40 -- STEP: 7900/16928 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.037035275250673294 (0.04125904988638948) + | > loss_mel_ce: 4.02374792098999 (3.861899348301222) + | > loss: 1.0151958465576172 (0.9757895999241486) + | > current_lr: 5e-06 + | > step_time: 0.2073 (0.32024102802517995) + | > loader_time: 0.0087 (0.009847205650957303) + + + --> TIME: 2024-10-20 05:38:08 -- STEP: 7950/16928 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.038363371044397354 (0.041253890999645416) + | > loss_mel_ce: 3.8479955196380615 (3.8610772419575605) + | > loss: 0.9715897440910339 (0.9755827836195635) + | > current_lr: 5e-06 + | > step_time: 0.3003 (0.3204098650194559) + | > loader_time: 0.0085 (0.009843573120405067) + + + --> TIME: 2024-10-20 05:38:35 -- STEP: 8000/16928 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.0406096987426281 (0.04124263405846418) + | > loss_mel_ce: 3.563894748687744 (3.860552116930483) + | > loss: 0.9011260867118835 (0.9754486880898482) + | > current_lr: 5e-06 + | > step_time: 0.2435 (0.3203896681666369) + | > loader_time: 0.0093 (0.00983971893787382) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_8000.pth + + --> TIME: 2024-10-20 05:39:16 -- STEP: 8050/16928 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.035275898873806 (0.04123030692292652) + | > loss_mel_ce: 3.4196717739105225 (3.859984060370402) + | > loss: 0.8637369275093079 (0.9753035921413713) + | > current_lr: 5e-06 + | > step_time: 0.3396 (0.3203658754336903) + | > loader_time: 0.0083 (0.009837208919643594) + + + --> TIME: 2024-10-20 05:39:41 -- STEP: 8100/16928 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.037463270127773285 (0.04121767097762139) + | > loss_mel_ce: 3.8496031761169434 (3.8589858695901453) + | > loss: 0.9717665910720825 (0.9750508854492218) + | > current_lr: 5e-06 + | > step_time: 0.2884 (0.3201777210353328) + | > loader_time: 0.0097 (0.009831649697857117) + + + --> TIME: 2024-10-20 05:40:07 -- STEP: 8150/16928 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.03941653296351433 (0.041205724060215325) + | > loss_mel_ce: 4.0874247550964355 (3.8586555933806044) + | > loss: 1.0317102670669556 (0.9749653296529156) + | > current_lr: 5e-06 + | > step_time: 0.2271 (0.32012316051436296) + | > loader_time: 0.0085 (0.00982937426654837) + + + --> TIME: 2024-10-20 05:40:34 -- STEP: 8200/16928 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.037319671362638474 (0.04119622299243205) + | > loss_mel_ce: 4.062371730804443 (3.858472921964597) + | > loss: 1.0249228477478027 (0.9749172865690261) + | > current_lr: 5e-06 + | > step_time: 0.2528 (0.3201466174241968) + | > loader_time: 0.0091 (0.009825628937744493) + + + --> TIME: 2024-10-20 05:41:01 -- STEP: 8250/16928 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.04219522327184677 (0.04118531907485276) + | > loss_mel_ce: 3.6237761974334717 (3.858226601571746) + | > loss: 0.9164928793907166 (0.9748529804764379) + | > current_lr: 5e-06 + | > step_time: 0.4946 (0.32016715962959) + | > loader_time: 0.0089 (0.00982345725550794) + + + --> TIME: 2024-10-20 05:41:27 -- STEP: 8300/16928 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.03603867441415787 (0.041179601523845526) + | > loss_mel_ce: 3.7029902935028076 (3.8577110074514342) + | > loss: 0.9347572326660156 (0.9747226525717477) + | > current_lr: 5e-06 + | > step_time: 0.2883 (0.3200907240143736) + | > loader_time: 0.0094 (0.009819459168307731) + + + --> TIME: 2024-10-20 05:41:54 -- STEP: 8350/16928 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.03682991489768028 (0.041169090740560986) + | > loss_mel_ce: 3.6165835857391357 (3.8573792164339973) + | > loss: 0.9133533835411072 (0.9746370771045462) + | > current_lr: 5e-06 + | > step_time: 0.3763 (0.32007825351760716) + | > loader_time: 0.0102 (0.009817880070851931) + + + --> TIME: 2024-10-20 05:42:23 -- STEP: 8400/16928 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.03868008404970169 (0.04115625150036052) + | > loss_mel_ce: 3.2914767265319824 (3.8568091628381165) + | > loss: 0.8325392007827759 (0.9744913539006602) + | > current_lr: 5e-06 + | > step_time: 0.3306 (0.3202449660641801) + | > loader_time: 0.0091 (0.009814307718049892) + + + --> TIME: 2024-10-20 05:42:51 -- STEP: 8450/16928 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.03797156363725662 (0.041147959424325906) + | > loss_mel_ce: 3.388770341873169 (3.856300676390968) + | > loss: 0.8566854596138 (0.9743621592761501) + | > current_lr: 5e-06 + | > step_time: 0.5039 (0.3203739068070807) + | > loader_time: 0.0095 (0.009811211083767661) + + + --> TIME: 2024-10-20 05:43:16 -- STEP: 8500/16928 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.04801664873957634 (0.041139237585970434) + | > loss_mel_ce: 3.841625928878784 (3.8555646855971393) + | > loss: 0.9724106192588806 (0.9741759811078805) + | > current_lr: 5e-06 + | > step_time: 0.2094 (0.3202501251276797) + | > loader_time: 0.01 (0.009806207937352778) + + + --> TIME: 2024-10-20 05:43:45 -- STEP: 8550/16928 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.04280824959278107 (0.04113341256082914) + | > loss_mel_ce: 4.04692268371582 (3.8551090707834703) + | > loss: 1.0224326848983765 (0.9740606211361137) + | > current_lr: 5e-06 + | > step_time: 0.3375 (0.3203543656053594) + | > loader_time: 0.0086 (0.009804032643636046) + + + --> TIME: 2024-10-20 05:44:12 -- STEP: 8600/16928 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.03979865834116936 (0.041125086934717094) + | > loss_mel_ce: 3.9638946056365967 (3.854525309939717) + | > loss: 1.0009232759475708 (0.9739125994956774) + | > current_lr: 5e-06 + | > step_time: 0.3994 (0.32036740702252) + | > loader_time: 0.0105 (0.009800682650055976) + + + --> TIME: 2024-10-20 05:44:39 -- STEP: 8650/16928 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.04040924832224846 (0.04111952965982201) + | > loss_mel_ce: 3.93041729927063 (3.8542138043166587) + | > loss: 0.9927066564559937 (0.9738333337775549) + | > current_lr: 5e-06 + | > step_time: 0.3099 (0.3203927237174409) + | > loader_time: 0.0096 (0.00979688867668194) + + + --> TIME: 2024-10-20 05:45:06 -- STEP: 8700/16928 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.03848707303404808 (0.041110972943447975) + | > loss_mel_ce: 3.67600679397583 (3.853705009701608) + | > loss: 0.9286234378814697 (0.973703995949921) + | > current_lr: 5e-06 + | > step_time: 0.2873 (0.32039453235165766) + | > loader_time: 0.0083 (0.009792989862376228) + + + --> TIME: 2024-10-20 05:45:32 -- STEP: 8750/16928 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.043258123099803925 (0.04110382073074563) + | > loss_mel_ce: 3.9139413833618164 (3.853030184882028) + | > loss: 0.9892998933792114 (0.9735335016795571) + | > current_lr: 5e-06 + | > step_time: 0.3355 (0.320331856155395) + | > loader_time: 0.0088 (0.00978827686309813) + + + --> TIME: 2024-10-20 05:45:59 -- STEP: 8800/16928 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.037598785012960434 (0.04109671697634342) + | > loss_mel_ce: 3.9116508960723877 (3.85272076476704) + | > loss: 0.9873124361038208 (0.9734543706883088) + | > current_lr: 5e-06 + | > step_time: 0.2503 (0.3203543335470279) + | > loader_time: 0.0092 (0.009784255054864008) + + + --> TIME: 2024-10-20 05:46:26 -- STEP: 8850/16928 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.04129483550786972 (0.0410879243870322) + | > loss_mel_ce: 4.1431097984313965 (3.851980735487857) + | > loss: 1.0461012125015259 (0.9732671652473303) + | > current_lr: 5e-06 + | > step_time: 0.2847 (0.32036939887677135) + | > loader_time: 0.0102 (0.009780234293749095) + + + --> TIME: 2024-10-20 05:46:53 -- STEP: 8900/16928 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.03731744363903999 (0.041079375046315654) + | > loss_mel_ce: 3.638500213623047 (3.8515039368157975) + | > loss: 0.9189544320106506 (0.9731458282738593) + | > current_lr: 5e-06 + | > step_time: 0.2248 (0.3203805036759101) + | > loader_time: 0.0098 (0.009775762879446643) + + + --> TIME: 2024-10-20 05:47:18 -- STEP: 8950/16928 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.039255011826753616 (0.04106954717457111) + | > loss_mel_ce: 3.4352903366088867 (3.8509809017181396) + | > loss: 0.8686363101005554 (0.9730126125519506) + | > current_lr: 5e-06 + | > step_time: 0.2829 (0.32024455065167723) + | > loader_time: 0.0078 (0.009770822871330725) + + + --> TIME: 2024-10-20 05:47:44 -- STEP: 9000/16928 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.039420515298843384 (0.04105970556144068) + | > loss_mel_ce: 3.478473663330078 (3.850367690934075) + | > loss: 0.8794735670089722 (0.9728568494253692) + | > current_lr: 5e-06 + | > step_time: 0.2445 (0.3201942697630979) + | > loader_time: 0.0094 (0.009768165508906041) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_9000.pth + + --> TIME: 2024-10-20 05:48:21 -- STEP: 9050/16928 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.041910793632268906 (0.041050011330587596) + | > loss_mel_ce: 3.5460755825042725 (3.8499122093000464) + | > loss: 0.8969966173171997 (0.9727405554681856) + | > current_lr: 5e-06 + | > step_time: 0.2829 (0.32011544517390556) + | > loader_time: 0.0099 (0.00976698933385353) + + + --> TIME: 2024-10-20 05:48:45 -- STEP: 9100/16928 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.03978613018989563 (0.04103811404063954) + | > loss_mel_ce: 3.691220283508301 (3.849528066116375) + | > loss: 0.9327515959739685 (0.9726415453507354) + | > current_lr: 5e-06 + | > step_time: 0.3027 (0.31992761467839254) + | > loader_time: 0.0094 (0.009763733423673185) + + + --> TIME: 2024-10-20 05:49:13 -- STEP: 9150/16928 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.0407959446310997 (0.041031109662777225) + | > loss_mel_ce: 3.842973470687866 (3.8491387263803536) + | > loss: 0.9709423780441284 (0.9725424593198498) + | > current_lr: 5e-06 + | > step_time: 0.3585 (0.3200088756071406) + | > loader_time: 0.0089 (0.009760099004526603) + + + --> TIME: 2024-10-20 05:49:39 -- STEP: 9200/16928 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.04202939569950104 (0.041021630995015865) + | > loss_mel_ce: 3.890084981918335 (3.8486009872218836) + | > loss: 0.9830285906791687 (0.9724056548528053) + | > current_lr: 5e-06 + | > step_time: 0.2093 (0.3199148312081453) + | > loader_time: 0.0086 (0.009757048228512635) + + + --> TIME: 2024-10-20 05:50:05 -- STEP: 9250/16928 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.038913507014513016 (0.04101344666106455) + | > loss_mel_ce: 3.9818501472473145 (3.8484351710242195) + | > loss: 1.0051909685134888 (0.9723621547157704) + | > current_lr: 5e-06 + | > step_time: 0.4434 (0.31986583859211615) + | > loader_time: 0.0092 (0.009751772700129325) + + + --> TIME: 2024-10-20 05:50:32 -- STEP: 9300/16928 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.03944295272231102 (0.04100444893502879) + | > loss_mel_ce: 3.633955955505371 (3.8480814188013794) + | > loss: 0.9183497428894043 (0.972271467235781) + | > current_lr: 5e-06 + | > step_time: 0.3446 (0.3198866523209433) + | > loader_time: 0.0097 (0.009748962284416272) + + + --> TIME: 2024-10-20 05:50:59 -- STEP: 9350/16928 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.03644286096096039 (0.04099576799049556) + | > loss_mel_ce: 3.6574511528015137 (3.8476171920133786) + | > loss: 0.9234734773635864 (0.9721532402637812) + | > current_lr: 5e-06 + | > step_time: 0.2494 (0.3199032698850578) + | > loader_time: 0.0087 (0.0097468428178267) + + + --> TIME: 2024-10-20 05:51:25 -- STEP: 9400/16928 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.041635897010564804 (0.04098763621055521) + | > loss_mel_ce: 3.6905646324157715 (3.8465304946899415) + | > loss: 0.9330501556396484 (0.9718795330093266) + | > current_lr: 5e-06 + | > step_time: 0.2485 (0.31986278617635655) + | > loader_time: 0.0079 (0.009743289262690436) + + + --> TIME: 2024-10-20 05:51:52 -- STEP: 9450/16928 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.039230238646268845 (0.04097246481214073) + | > loss_mel_ce: 4.048274517059326 (3.8458029448544537) + | > loss: 1.0218762159347534 (0.9716938527046692) + | > current_lr: 5e-06 + | > step_time: 0.4641 (0.31985054778043526) + | > loader_time: 0.0097 (0.009740331816294832) + + + --> TIME: 2024-10-20 05:52:19 -- STEP: 9500/16928 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.042649220675230026 (0.04096100743879594) + | > loss_mel_ce: 3.5787911415100098 (3.8451645660149425) + | > loss: 0.9053601026535034 (0.9715313936409201) + | > current_lr: 5e-06 + | > step_time: 0.3467 (0.31990293570568656) + | > loader_time: 0.0092 (0.00973821773027118) + + + --> TIME: 2024-10-20 05:52:48 -- STEP: 9550/16928 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.03784611448645592 (0.040952407476048294) + | > loss_mel_ce: 3.696056604385376 (3.8448329365815166) + | > loss: 0.9334756731987 (0.971446336301834) + | > current_lr: 5e-06 + | > step_time: 0.2467 (0.32005669311703183) + | > loader_time: 0.0084 (0.009736511245447915) + + + --> TIME: 2024-10-20 05:53:17 -- STEP: 9600/16928 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.03808319568634033 (0.040942341799187876) + | > loss_mel_ce: 3.995849609375 (3.844289981275797) + | > loss: 1.0084831714630127 (0.9713080810755496) + | > current_lr: 5e-06 + | > step_time: 0.3554 (0.3201484822978573) + | > loader_time: 0.0096 (0.009733890419205015) + + + --> TIME: 2024-10-20 05:53:43 -- STEP: 9650/16928 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.03811430186033249 (0.040935217803585194) + | > loss_mel_ce: 3.8345353603363037 (3.8437928618791806) + | > loss: 0.9681624174118042 (0.9711820202232029) + | > current_lr: 5e-06 + | > step_time: 0.2116 (0.32011509593904275) + | > loader_time: 0.0082 (0.009730356517851028) + + + --> TIME: 2024-10-20 05:54:11 -- STEP: 9700/16928 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.040868859738111496 (0.04093051863488596) + | > loss_mel_ce: 3.8094050884246826 (3.84348862785654) + | > loss: 0.962568461894989 (0.9711047868999013) + | > current_lr: 5e-06 + | > step_time: 0.2527 (0.3201980624985443) + | > loader_time: 0.0083 (0.009727662174972045) + + + --> TIME: 2024-10-20 05:54:38 -- STEP: 9750/16928 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.037179987877607346 (0.04092167733399509) + | > loss_mel_ce: 4.253783702850342 (3.8433569644292196) + | > loss: 1.072740912437439 (0.9710696607125114) + | > current_lr: 5e-06 + | > step_time: 0.26 (0.32019106212029025) + | > loader_time: 0.0083 (0.009726148531987099) + + + --> TIME: 2024-10-20 05:55:05 -- STEP: 9800/16928 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.041870858520269394 (0.040911629917099845) + | > loss_mel_ce: 4.260540008544922 (3.843112141112892) + | > loss: 1.0756027698516846 (0.9710059430222125) + | > current_lr: 5e-06 + | > step_time: 0.2019 (0.32024121615351386) + | > loader_time: 0.0092 (0.009722193284910533) + + + --> TIME: 2024-10-20 05:55:33 -- STEP: 9850/16928 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.03704386577010155 (0.04090266416652986) + | > loss_mel_ce: 3.6707346439361572 (3.8426686081910497) + | > loss: 0.9269446134567261 (0.9708928183734723) + | > current_lr: 5e-06 + | > step_time: 0.4687 (0.32032287011896815) + | > loader_time: 0.0095 (0.009718300461163959) + + + --> TIME: 2024-10-20 05:56:01 -- STEP: 9900/16928 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.03921657055616379 (0.04089732582878413) + | > loss_mel_ce: 4.041407585144043 (3.8418930267083526) + | > loss: 1.0201560258865356 (0.9706975884268986) + | > current_lr: 5e-06 + | > step_time: 0.2605 (0.3203667450191989) + | > loader_time: 0.009 (0.009715037996118699) + + + --> TIME: 2024-10-20 05:56:25 -- STEP: 9950/16928 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.04290591925382614 (0.040890207229434636) + | > loss_mel_ce: 3.8492772579193115 (3.8412702206271376) + | > loss: 0.9730457663536072 (0.9705401072550063) + | > current_lr: 5e-06 + | > step_time: 0.3339 (0.3202378130677947) + | > loader_time: 0.0087 (0.009711537696608328) + + + --> TIME: 2024-10-20 05:56:53 -- STEP: 10000/16928 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.04065786674618721 (0.040877832045964686) + | > loss_mel_ce: 3.833099842071533 (3.840569262766838) + | > loss: 0.9684394001960754 (0.9703617740035061) + | > current_lr: 5e-06 + | > step_time: 0.2705 (0.3203009665250771) + | > loader_time: 0.0096 (0.009709542250633215) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_10000.pth + + --> TIME: 2024-10-20 05:57:32 -- STEP: 10050/16928 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.03843547776341438 (0.04086790888185652) + | > loss_mel_ce: 3.929419755935669 (3.839558465801068) + | > loss: 0.991963803768158 (0.9701065939575882) + | > current_lr: 5e-06 + | > step_time: 0.2769 (0.320275818340813) + | > loader_time: 0.0104 (0.009710277229992287) + + + --> TIME: 2024-10-20 05:57:58 -- STEP: 10100/16928 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.040895190089941025 (0.04086033085508639) + | > loss_mel_ce: 4.161677837371826 (3.839007296562195) + | > loss: 1.0506432056427002 (0.9699669071117254) + | > current_lr: 5e-06 + | > step_time: 0.2358 (0.3202406582501847) + | > loader_time: 0.0086 (0.009706966074386423) + + + --> TIME: 2024-10-20 05:58:23 -- STEP: 10150/16928 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.03468328341841698 (0.04085071263293453) + | > loss_mel_ce: 3.382443904876709 (3.838586712705678) + | > loss: 0.8542817831039429 (0.969859356610059) + | > current_lr: 5e-06 + | > step_time: 0.2484 (0.32013473973485546) + | > loader_time: 0.0078 (0.009703315819425509) + + + --> TIME: 2024-10-20 05:58:50 -- STEP: 10200/16928 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.038527343422174454 (0.040840808456836464) + | > loss_mel_ce: 3.7889201641082764 (3.8379152888643975) + | > loss: 0.9568618535995483 (0.9696890246166906) + | > current_lr: 5e-06 + | > step_time: 0.2467 (0.32010445765420414) + | > loader_time: 0.0088 (0.009701173656127006) + + + --> TIME: 2024-10-20 05:59:17 -- STEP: 10250/16928 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.0394587479531765 (0.04083513122088278) + | > loss_mel_ce: 3.7511274814605713 (3.8375681706870473) + | > loss: 0.9476465582847595 (0.9696008257865909) + | > current_lr: 5e-06 + | > step_time: 0.3131 (0.3201429425914105) + | > loader_time: 0.0082 (0.009699375524753454) + + + --> TIME: 2024-10-20 05:59:46 -- STEP: 10300/16928 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.038786884397268295 (0.040826378939401844) + | > loss_mel_ce: 3.387326717376709 (3.8367006943526776) + | > loss: 0.8565284013748169 (0.9693817686224451) + | > current_lr: 5e-06 + | > step_time: 0.2093 (0.3202273004957764) + | > loader_time: 0.0086 (0.009698998580858527) + + + --> TIME: 2024-10-20 06:00:13 -- STEP: 10350/16928 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.036439262330532074 (0.04081793152317747) + | > loss_mel_ce: 3.51102614402771 (3.836536407056062) + | > loss: 0.8868663311004639 (0.9693385849597953) + | > current_lr: 5e-06 + | > step_time: 0.338 (0.32027500765335076) + | > loader_time: 0.0099 (0.009697197324411867) + + + --> TIME: 2024-10-20 06:00:40 -- STEP: 10400/16928 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.03904169797897339 (0.04081393095825854) + | > loss_mel_ce: 3.5152840614318848 (3.8363728803396224) + | > loss: 0.8885814547538757 (0.9692967031552245) + | > current_lr: 5e-06 + | > step_time: 0.3728 (0.320259628089574) + | > loader_time: 0.0096 (0.009696204524773804) + + + --> TIME: 2024-10-20 06:01:06 -- STEP: 10450/16928 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.035412661731243134 (0.04080573886258592) + | > loss_mel_ce: 3.6453964710235596 (3.8358931946184076) + | > loss: 0.9202022552490234 (0.9691747336809718) + | > current_lr: 5e-06 + | > step_time: 0.1907 (0.32022154296984423) + | > loader_time: 0.0093 (0.009694639666799138) + + + --> TIME: 2024-10-20 06:01:34 -- STEP: 10500/16928 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.03977181389927864 (0.04079793561125777) + | > loss_mel_ce: 3.437378406524658 (3.835502601578122) + | > loss: 0.8692875504493713 (0.9690751345952355) + | > current_lr: 5e-06 + | > step_time: 0.2708 (0.3203049401101601) + | > loader_time: 0.009 (0.00969388228371028) + + + --> TIME: 2024-10-20 06:02:01 -- STEP: 10550/16928 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.03561108931899071 (0.040788338820833825) + | > loss_mel_ce: 3.8300390243530273 (3.8353458891773675) + | > loss: 0.9664125442504883 (0.9690335573178336) + | > current_lr: 5e-06 + | > step_time: 0.3589 (0.3202726625605205) + | > loader_time: 0.01 (0.009691508234394654) + + + --> TIME: 2024-10-20 06:02:28 -- STEP: 10600/16928 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.039250448346138 (0.0407806072101206) + | > loss_mel_ce: 3.404022693634033 (3.8345650012313195) + | > loss: 0.8608182668685913 (0.9688364024151049) + | > current_lr: 5e-06 + | > step_time: 0.3567 (0.32030052182809243) + | > loader_time: 0.009 (0.009688476616481548) + + + --> TIME: 2024-10-20 06:02:55 -- STEP: 10650/16928 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.03572583198547363 (0.040770822436211905) + | > loss_mel_ce: 3.8665220737457275 (3.8345242506125725) + | > loss: 0.9755619764328003 (0.9688237685608755) + | > current_lr: 5e-06 + | > step_time: 0.3618 (0.3203515876178999) + | > loader_time: 0.0093 (0.009687611485870776) + + + --> TIME: 2024-10-20 06:03:22 -- STEP: 10700/16928 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.04125458747148514 (0.04076496619609316) + | > loss_mel_ce: 3.6009318828582764 (3.8338595343973036) + | > loss: 0.910546600818634 (0.9686561254251786) + | > current_lr: 5e-06 + | > step_time: 0.2087 (0.32034752876959016) + | > loader_time: 0.009 (0.009686266596072165) + + + --> TIME: 2024-10-20 06:03:49 -- STEP: 10750/16928 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.04487121105194092 (0.040757230300716034) + | > loss_mel_ce: 3.6635754108428955 (3.8334985146189844) + | > loss: 0.9271116256713867 (0.9685639365052071) + | > current_lr: 5e-06 + | > step_time: 0.3352 (0.3203366660628197) + | > loader_time: 0.0091 (0.009684367579083083) + + + --> TIME: 2024-10-20 06:04:17 -- STEP: 10800/16928 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.03582358360290527 (0.04075053820541731) + | > loss_mel_ce: 3.6463427543640137 (3.8329841734524126) + | > loss: 0.9205415844917297 (0.9684336781722531) + | > current_lr: 5e-06 + | > step_time: 0.1852 (0.32038118470598037) + | > loader_time: 0.0087 (0.009682344043696339) + + + --> TIME: 2024-10-20 06:04:43 -- STEP: 10850/16928 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.03254597261548042 (0.04074136953938236) + | > loss_mel_ce: 4.298949718475342 (3.832644861590478) + | > loss: 1.0828739404678345 (0.9683465580182145) + | > current_lr: 5e-06 + | > step_time: 0.2605 (0.3203551818258741) + | > loader_time: 0.0092 (0.009681558762827194) + + + --> TIME: 2024-10-20 06:05:08 -- STEP: 10900/16928 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.03909018635749817 (0.04073193305419791) + | > loss_mel_ce: 3.4611704349517822 (3.8320248032709876) + | > loss: 0.8750651478767395 (0.9681891843375813) + | > current_lr: 5e-06 + | > step_time: 0.2442 (0.32026036350005155) + | > loader_time: 0.0086 (0.009680876031928078) + + + --> TIME: 2024-10-20 06:05:37 -- STEP: 10950/16928 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.03880768269300461 (0.04072187017309183) + | > loss_mel_ce: 3.8639891147613525 (3.831648750762417) + | > loss: 0.9756991863250732 (0.9680926554878012) + | > current_lr: 5e-06 + | > step_time: 0.2557 (0.3203549940509872) + | > loader_time: 0.0088 (0.009680310580284056) + + + --> TIME: 2024-10-20 06:06:03 -- STEP: 11000/16928 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.035877399146556854 (0.04071293565139831) + | > loss_mel_ce: 4.014404296875 (3.8314392740076237) + | > loss: 1.0125703811645508 (0.9680380526726899) + | > current_lr: 5e-06 + | > step_time: 0.2032 (0.32029123741930066) + | > loader_time: 0.0091 (0.009679678591814879) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_11000.pth + + --> TIME: 2024-10-20 06:06:39 -- STEP: 11050/16928 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.03663241118192673 (0.04070561043698976) + | > loss_mel_ce: 4.389030933380127 (3.831028797162604) + | > loss: 1.106415867805481 (0.9679336021496703) + | > current_lr: 5e-06 + | > step_time: 0.3424 (0.3202289661778573) + | > loader_time: 0.0084 (0.009679700670199127) + + + --> TIME: 2024-10-20 06:07:07 -- STEP: 11100/16928 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.04112676903605461 (0.040699650700463975) + | > loss_mel_ce: 3.7554996013641357 (3.830891359367886) + | > loss: 0.9491565823554993 (0.9678977527435839) + | > current_lr: 5e-06 + | > step_time: 0.2716 (0.32030053106514067) + | > loader_time: 0.0097 (0.009678598889359452) + + + --> TIME: 2024-10-20 06:07:32 -- STEP: 11150/16928 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.041805896908044815 (0.0406895083481352) + | > loss_mel_ce: 3.875471591949463 (3.8303334386038674) + | > loss: 0.9793193936347961 (0.9677557369518711) + | > current_lr: 5e-06 + | > step_time: 0.4408 (0.3202034348757267) + | > loader_time: 0.0094 (0.00967640019318445) + + + --> TIME: 2024-10-20 06:08:00 -- STEP: 11200/16928 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.03469843417406082 (0.04068239310739677) + | > loss_mel_ce: 3.731259346008301 (3.8298956042528154) + | > loss: 0.9414894580841064 (0.9676444995616167) + | > current_lr: 5e-06 + | > step_time: 0.2487 (0.3202534149161395) + | > loader_time: 0.0085 (0.00967509955167767) + + + --> TIME: 2024-10-20 06:08:27 -- STEP: 11250/16928 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.041664689779281616 (0.040674267208741614) + | > loss_mel_ce: 4.1694111824035645 (3.8295353073543974) + | > loss: 1.0527689456939697 (0.9675523938761821) + | > current_lr: 5e-06 + | > step_time: 0.3806 (0.32027156806521845) + | > loader_time: 0.0104 (0.009672589174906373) + + + --> TIME: 2024-10-20 06:08:53 -- STEP: 11300/16928 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.035758357495069504 (0.04066741079009592) + | > loss_mel_ce: 3.8490939140319824 (3.8291097495830164) + | > loss: 0.9712130427360535 (0.9674442903183206) + | > current_lr: 5e-06 + | > step_time: 0.4165 (0.3202348565844297) + | > loader_time: 0.0103 (0.009671199933617479) + + + --> TIME: 2024-10-20 06:09:22 -- STEP: 11350/16928 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.040792010724544525 (0.04066060115735029) + | > loss_mel_ce: 3.6330151557922363 (3.8287346562204907) + | > loss: 0.9184517860412598 (0.967348814551526) + | > current_lr: 5e-06 + | > step_time: 0.5046 (0.3203456033168901) + | > loader_time: 0.0099 (0.009669708575446138) + + + --> TIME: 2024-10-20 06:09:50 -- STEP: 11400/16928 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.038241248577833176 (0.04065390765160439) + | > loss_mel_ce: 3.949443817138672 (3.8284200348770407) + | > loss: 0.9969212412834167 (0.9672684858085819) + | > current_lr: 5e-06 + | > step_time: 0.2485 (0.32041039571427377) + | > loader_time: 0.0086 (0.009668367661927846) + + + --> TIME: 2024-10-20 06:10:17 -- STEP: 11450/16928 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.03802482411265373 (0.0406432923412673) + | > loss_mel_ce: 3.8590192794799805 (3.8280372944461205) + | > loss: 0.9742610454559326 (0.9671701468807123) + | > current_lr: 5e-06 + | > step_time: 0.443 (0.32042839173146137) + | > loader_time: 0.0089 (0.009667640877602895) + + + --> TIME: 2024-10-20 06:10:45 -- STEP: 11500/16928 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.03434397280216217 (0.04063585833379092) + | > loss_mel_ce: 3.952213764190674 (3.8279072786828747) + | > loss: 0.9966394305229187 (0.967135784434236) + | > current_lr: 5e-06 + | > step_time: 0.3706 (0.3204739164476802) + | > loader_time: 0.0103 (0.009664968656456966) + + + --> TIME: 2024-10-20 06:11:14 -- STEP: 11550/16928 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.03942997753620148 (0.040630510188639) + | > loss_mel_ce: 3.4558634757995605 (3.827183540889195) + | > loss: 0.873823344707489 (0.9669535129462489) + | > current_lr: 5e-06 + | > step_time: 0.3352 (0.32060892526205415) + | > loader_time: 0.0085 (0.00966427615194608) + + + --> TIME: 2024-10-20 06:11:41 -- STEP: 11600/16928 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.042161159217357635 (0.04062246175531026) + | > loss_mel_ce: 3.436056137084961 (3.8265957356732465) + | > loss: 0.869554340839386 (0.96680454954505) + | > current_lr: 5e-06 + | > step_time: 0.183 (0.3206344960064717) + | > loader_time: 0.0093 (0.009663238196537393) + + + --> TIME: 2024-10-20 06:12:07 -- STEP: 11650/16928 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.03572715073823929 (0.040616403125674) + | > loss_mel_ce: 3.974585771560669 (3.825923715067524) + | > loss: 1.0025782585144043 (0.9666350297405999) + | > current_lr: 5e-06 + | > step_time: 0.4966 (0.3205650392417737) + | > loader_time: 0.0098 (0.009661204067934215) + + + --> TIME: 2024-10-20 06:12:33 -- STEP: 11700/16928 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.03641160577535629 (0.040608070566772486) + | > loss_mel_ce: 3.5132362842559814 (3.8255090400092624) + | > loss: 0.8874119520187378 (0.9665292778229105) + | > current_lr: 5e-06 + | > step_time: 0.2701 (0.3205302328533591) + | > loader_time: 0.0101 (0.009659403205936784) + + + --> TIME: 2024-10-20 06:13:00 -- STEP: 11750/16928 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.04030441492795944 (0.040599101395841435) + | > loss_mel_ce: 3.6291344165802 (3.825232024902993) + | > loss: 0.9173597097396851 (0.9664577817511054) + | > current_lr: 5e-06 + | > step_time: 0.2759 (0.32051710416915485) + | > loader_time: 0.0106 (0.009658156171758117) + + + --> TIME: 2024-10-20 06:13:26 -- STEP: 11800/16928 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.0387907549738884 (0.04059060028122755) + | > loss_mel_ce: 3.639864206314087 (3.824905934616671) + | > loss: 0.9196637272834778 (0.9663741338859173) + | > current_lr: 5e-06 + | > step_time: 0.3735 (0.3204355848441681) + | > loader_time: 0.0093 (0.009656453839803144) + + + --> TIME: 2024-10-20 06:13:51 -- STEP: 11850/16928 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.039319925010204315 (0.040580643444844564) + | > loss_mel_ce: 3.750298023223877 (3.8243579298132078) + | > loss: 0.9474045038223267 (0.9662346434744101) + | > current_lr: 5e-06 + | > step_time: 0.2579 (0.32035386286707274) + | > loader_time: 0.0099 (0.009654884438977464) + + + --> TIME: 2024-10-20 06:14:15 -- STEP: 11900/16928 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.03514504432678223 (0.04057096792510191) + | > loss_mel_ce: 3.475670099258423 (3.8239907732731155) + | > loss: 0.8777037858963013 (0.9661404354532229) + | > current_lr: 5e-06 + | > step_time: 0.2027 (0.3201878296026655) + | > loader_time: 0.0079 (0.009651323486776917) + + + --> TIME: 2024-10-20 06:14:41 -- STEP: 11950/16928 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.039212994277477264 (0.040563135773223964) + | > loss_mel_ce: 3.9773952960968018 (3.8236963467019374) + | > loss: 1.0041520595550537 (0.9660648707515528) + | > current_lr: 5e-06 + | > step_time: 0.3086 (0.3201573455682849) + | > loader_time: 0.0082 (0.009649406596706508) + + + --> TIME: 2024-10-20 06:15:08 -- STEP: 12000/16928 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.03935518115758896 (0.04055394738803918) + | > loss_mel_ce: 3.6664681434631348 (3.8233137002388635) + | > loss: 0.9264558553695679 (0.9659669120460752) + | > current_lr: 5e-06 + | > step_time: 0.2249 (0.32013096143801895) + | > loader_time: 0.009 (0.009647609372933713) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_12000.pth + + --> TIME: 2024-10-20 06:15:47 -- STEP: 12050/16928 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.03615009784698486 (0.04054760234499981) + | > loss_mel_ce: 3.762072801589966 (3.822965547415231) + | > loss: 0.9495557546615601 (0.965878287593359) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.32018925320557945) + | > loader_time: 0.0104 (0.009647491878493704) + + + --> TIME: 2024-10-20 06:16:13 -- STEP: 12100/16928 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.036858342587947845 (0.040537716825300094) + | > loss_mel_ce: 3.532972574234009 (3.8226500691461167) + | > loss: 0.8924577236175537 (0.9657969466516797) + | > current_lr: 5e-06 + | > step_time: 0.2716 (0.32016635031739465) + | > loader_time: 0.0086 (0.00964672431472906) + + + --> TIME: 2024-10-20 06:16:39 -- STEP: 12150/16928 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.04234215244650841 (0.040527797440840965) + | > loss_mel_ce: 3.8257040977478027 (3.822003988003044) + | > loss: 0.967011570930481 (0.9656329465353933) + | > current_lr: 5e-06 + | > step_time: 0.3148 (0.32010479709248435) + | > loader_time: 0.0102 (0.009644010959829337) + + + --> TIME: 2024-10-20 06:17:07 -- STEP: 12200/16928 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.041409075260162354 (0.04051928481896258) + | > loss_mel_ce: 3.187685966491699 (3.821573655370806) + | > loss: 0.8072737455368042 (0.9655232352164929) + | > current_lr: 5e-06 + | > step_time: 0.3576 (0.3201526182596797) + | > loader_time: 0.01 (0.009643670183713334) + + + --> TIME: 2024-10-20 06:17:33 -- STEP: 12250/16928 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.036755215376615524 (0.04050991148988191) + | > loss_mel_ce: 4.221371650695801 (3.82128696268432) + | > loss: 1.064531683921814 (0.9654492187110748) + | > current_lr: 5e-06 + | > step_time: 0.3364 (0.32009346276886574) + | > loader_time: 0.0103 (0.009642122346527745) + + + --> TIME: 2024-10-20 06:17:58 -- STEP: 12300/16928 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.03752188757061958 (0.04050257774054754) + | > loss_mel_ce: 3.3078291416168213 (3.820689691702525) + | > loss: 0.8363377451896667 (0.9652980675203047) + | > current_lr: 5e-06 + | > step_time: 0.2446 (0.32001667611967227) + | > loader_time: 0.0102 (0.009640428961777121) + + + --> TIME: 2024-10-20 06:18:24 -- STEP: 12350/16928 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.03859593719244003 (0.04049441342532383) + | > loss_mel_ce: 3.6586410999298096 (3.8203854264131922) + | > loss: 0.924309253692627 (0.9652199601113558) + | > current_lr: 5e-06 + | > step_time: 0.3116 (0.319924728629077) + | > loader_time: 0.0094 (0.009639214349661779) + + + --> TIME: 2024-10-20 06:18:52 -- STEP: 12400/16928 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.039331451058387756 (0.040484767384947204) + | > loss_mel_ce: 3.7559762001037598 (3.8198594727054718) + | > loss: 0.9488269090652466 (0.965086060171166) + | > current_lr: 5e-06 + | > step_time: 0.3207 (0.3200023881850701) + | > loader_time: 0.0099 (0.009638799717349406) + + + --> TIME: 2024-10-20 06:19:19 -- STEP: 12450/16928 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.037577711045742035 (0.040476524713767) + | > loss_mel_ce: 3.985856533050537 (3.819485277593375) + | > loss: 1.0058585405349731 (0.9649904507231045) + | > current_lr: 5e-06 + | > step_time: 0.3239 (0.3200275192873544) + | > loader_time: 0.0103 (0.00963767155107247) + + + --> TIME: 2024-10-20 06:19:47 -- STEP: 12500/16928 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.038278304040431976 (0.0404687240588664) + | > loss_mel_ce: 3.6891963481903076 (3.819350892391205) + | > loss: 0.9318686723709106 (0.9649549042463306) + | > current_lr: 5e-06 + | > step_time: 0.2752 (0.32003899709701533) + | > loader_time: 0.0092 (0.00963614290237428) + + + --> TIME: 2024-10-20 06:20:14 -- STEP: 12550/16928 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.03875447064638138 (0.04045998898605652) + | > loss_mel_ce: 3.8534297943115234 (3.8191739987947075) + | > loss: 0.973046064376831 (0.9649084971008076) + | > current_lr: 5e-06 + | > step_time: 0.4983 (0.3200686572177476) + | > loader_time: 0.0109 (0.00963525126179852) + + + --> TIME: 2024-10-20 06:20:40 -- STEP: 12600/16928 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.039982445538043976 (0.04045132846601058) + | > loss_mel_ce: 3.776123046875 (3.8188283388955253) + | > loss: 0.9540264010429382 (0.9648199170118289) + | > current_lr: 5e-06 + | > step_time: 0.3705 (0.3200310252772437) + | > loader_time: 0.0113 (0.009633122625805093) + + + --> TIME: 2024-10-20 06:21:07 -- STEP: 12650/16928 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.037293143570423126 (0.04044502539081639) + | > loss_mel_ce: 4.0659379959106445 (3.818425971943399) + | > loss: 1.0258077383041382 (0.9647177494938669) + | > current_lr: 5e-06 + | > step_time: 0.3812 (0.3200099240555594) + | > loader_time: 0.0097 (0.009631352217301088) + + + --> TIME: 2024-10-20 06:21:32 -- STEP: 12700/16928 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.03701934590935707 (0.04043769451004886) + | > loss_mel_ce: 3.606494426727295 (3.817818843886608) + | > loss: 0.9108784198760986 (0.9645641347648594) + | > current_lr: 5e-06 + | > step_time: 0.2457 (0.3199529059102217) + | > loader_time: 0.0087 (0.009629456396178007) + + + --> TIME: 2024-10-20 06:21:59 -- STEP: 12750/16928 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.0386798195540905 (0.04042850830055321) + | > loss_mel_ce: 3.4014408588409424 (3.8177652976653156) + | > loss: 0.8600301742553711 (0.9645484516667387) + | > current_lr: 5e-06 + | > step_time: 0.358 (0.31996620118384284) + | > loader_time: 0.0087 (0.009627216657002776) + + + --> TIME: 2024-10-20 06:22:27 -- STEP: 12800/16928 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.03559786081314087 (0.04042187278289916) + | > loss_mel_ce: 3.6604316234588623 (3.8177269115671515) + | > loss: 0.9240073561668396 (0.9645371962757784) + | > current_lr: 5e-06 + | > step_time: 0.2767 (0.3199731937237087) + | > loader_time: 0.0084 (0.009625037536025062) + + + --> TIME: 2024-10-20 06:22:53 -- STEP: 12850/16928 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.03780185058712959 (0.040412291480438345) + | > loss_mel_ce: 3.5974090099334717 (3.8174163402935872) + | > loss: 0.9088026881217957 (0.9644571581192984) + | > current_lr: 5e-06 + | > step_time: 0.4468 (0.31994043430001845) + | > loader_time: 0.0092 (0.009622616137048167) + + + --> TIME: 2024-10-20 06:23:20 -- STEP: 12900/16928 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.0376247800886631 (0.04040638284468937) + | > loss_mel_ce: 3.6326677799224854 (3.817154856559842) + | > loss: 0.9175731539726257 (0.9643903100351957) + | > current_lr: 5e-06 + | > step_time: 0.3199 (0.31993964171224815) + | > loader_time: 0.0088 (0.009620702414549607) + + + --> TIME: 2024-10-20 06:23:45 -- STEP: 12950/16928 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.0400874949991703 (0.04039902987181551) + | > loss_mel_ce: 3.746757745742798 (3.8165858959139083) + | > loss: 0.9467113018035889 (0.9642462316397077) + | > current_lr: 5e-06 + | > step_time: 0.2894 (0.31988796521337826) + | > loader_time: 0.0088 (0.009618878051581082) + + + --> TIME: 2024-10-20 06:24:12 -- STEP: 13000/16928 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.040582943707704544 (0.040391447429903284) + | > loss_mel_ce: 4.16204309463501 (3.816251378774643) + | > loss: 1.0506565570831299 (0.9641607067264046) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.31987771690808836) + | > loader_time: 0.0085 (0.009616322847513055) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_13000.pth + + --> TIME: 2024-10-20 06:24:50 -- STEP: 13050/16928 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.03639276698231697 (0.0403840482641738) + | > loss_mel_ce: 2.901195764541626 (3.8158287680103404) + | > loss: 0.7343971133232117 (0.9640532042559996) + | > current_lr: 5e-06 + | > step_time: 0.2997 (0.3198886596928157) + | > loader_time: 0.0084 (0.009616126192027126) + + + --> TIME: 2024-10-20 06:25:16 -- STEP: 13100/16928 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.03861387073993683 (0.040375645973556) + | > loss_mel_ce: 3.9586570262908936 (3.815436507836553) + | > loss: 0.9993177056312561 (0.963953038640605) + | > current_lr: 5e-06 + | > step_time: 0.3724 (0.3198458415133355) + | > loader_time: 0.0088 (0.009614491262508712) + + + --> TIME: 2024-10-20 06:25:44 -- STEP: 13150/16928 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.045252278447151184 (0.04036991622582234) + | > loss_mel_ce: 3.316930055618286 (3.814664984655924) + | > loss: 0.8405455946922302 (0.9637587254083658) + | > current_lr: 5e-06 + | > step_time: 0.2486 (0.31987747132551564) + | > loader_time: 0.0087 (0.009613266934006836) + + + --> TIME: 2024-10-20 06:26:10 -- STEP: 13200/16928 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.04110723361372948 (0.04036162800988112) + | > loss_mel_ce: 3.9791932106018066 (3.814266709125403) + | > loss: 1.0050750970840454 (0.9636570844776705) + | > current_lr: 5e-06 + | > step_time: 0.2362 (0.3198759746009656) + | > loader_time: 0.0085 (0.00961232338890884) + + + --> TIME: 2024-10-20 06:26:38 -- STEP: 13250/16928 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.03697831183671951 (0.040354463729796404) + | > loss_mel_ce: 4.0317277908325195 (3.8140835329991467) + | > loss: 1.0171765089035034 (0.9636094993825232) + | > current_lr: 5e-06 + | > step_time: 0.3345 (0.3199140925857259) + | > loader_time: 0.0077 (0.009611003048015087) + + + --> TIME: 2024-10-20 06:27:05 -- STEP: 13300/16928 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.03842714801430702 (0.040346464226652445) + | > loss_mel_ce: 3.5489630699157715 (3.8136592332223302) + | > loss: 0.8968475461006165 (0.9635014245608698) + | > current_lr: 5e-06 + | > step_time: 0.2089 (0.3198988570485799) + | > loader_time: 0.0099 (0.00960954691234388) + + + --> TIME: 2024-10-20 06:27:30 -- STEP: 13350/16928 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.03337882086634636 (0.04033691721895089) + | > loss_mel_ce: 3.7043087482452393 (3.8133043820670482) + | > loss: 0.9344218969345093 (0.963410325019101) + | > current_lr: 5e-06 + | > step_time: 0.227 (0.3198331079233008) + | > loader_time: 0.009 (0.009607400251238537) + + + --> TIME: 2024-10-20 06:27:57 -- STEP: 13400/16928 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.0357573963701725 (0.04033029022209555) + | > loss_mel_ce: 4.4381232261657715 (3.8129389528374174) + | > loss: 1.1184701919555664 (0.9633173109791174) + | > current_lr: 5e-06 + | > step_time: 0.2873 (0.31983894301884236) + | > loader_time: 0.009 (0.009605775253096627) + + + --> TIME: 2024-10-20 06:28:24 -- STEP: 13450/16928 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.039468973875045776 (0.04032424110476066) + | > loss_mel_ce: 3.8429043292999268 (3.8123927738586767) + | > loss: 0.9705933332443237 (0.9631792539513246) + | > current_lr: 5e-06 + | > step_time: 0.2507 (0.31982919090299383) + | > loader_time: 0.0084 (0.00960294509022653) + + + --> TIME: 2024-10-20 06:28:51 -- STEP: 13500/16928 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.038202617317438126 (0.040315228394749957) + | > loss_mel_ce: 3.467984676361084 (3.8118197820804736) + | > loss: 0.8765468001365662 (0.9630337528431859) + | > current_lr: 5e-06 + | > step_time: 0.2148 (0.319836837574288) + | > loader_time: 0.0081 (0.009601281642913823) + + + --> TIME: 2024-10-20 06:29:16 -- STEP: 13550/16928 -- GLOBAL_STEP: 13550 + | > loss_text_ce: 0.038397982716560364 (0.04030651016855028) + | > loss_mel_ce: 3.5287024974823 (3.8112800657441257) + | > loss: 0.8917751312255859 (0.9628966442069447) + | > current_lr: 5e-06 + | > step_time: 0.398 (0.3197900731977064) + | > loader_time: 0.0093 (0.009600006275951208) + + + --> TIME: 2024-10-20 06:29:44 -- STEP: 13600/16928 -- GLOBAL_STEP: 13600 + | > loss_text_ce: 0.04025529325008392 (0.040300308032481635) + | > loss_mel_ce: 3.531776189804077 (3.8109287574186044) + | > loss: 0.8930078744888306 (0.9628072665982389) + | > current_lr: 5e-06 + | > step_time: 0.3708 (0.31980905117357505) + | > loader_time: 0.0106 (0.009599572122097019) + + + --> TIME: 2024-10-20 06:30:10 -- STEP: 13650/16928 -- GLOBAL_STEP: 13650 + | > loss_text_ce: 0.038772597908973694 (0.04029380236230368) + | > loss_mel_ce: 4.088633060455322 (3.8104901551152324) + | > loss: 1.0318514108657837 (0.962695989595665) + | > current_lr: 5e-06 + | > step_time: 0.2777 (0.31979005139389344) + | > loader_time: 0.0081 (0.009598127896095812) + + + --> TIME: 2024-10-20 06:30:36 -- STEP: 13700/16928 -- GLOBAL_STEP: 13700 + | > loss_text_ce: 0.038357459008693695 (0.04028712897255569) + | > loss_mel_ce: 3.6706888675689697 (3.810214692067056) + | > loss: 0.9272615909576416 (0.9626254554995658) + | > current_lr: 5e-06 + | > step_time: 0.1839 (0.319773717587882) + | > loader_time: 0.0086 (0.009596296157280028) + + + --> TIME: 2024-10-20 06:31:03 -- STEP: 13750/16928 -- GLOBAL_STEP: 13750 + | > loss_text_ce: 0.03886939212679863 (0.04027999311075956) + | > loss_mel_ce: 3.8903939723968506 (3.809648196775263) + | > loss: 0.9823158383369446 (0.9624820477182218) + | > current_lr: 5e-06 + | > step_time: 0.4432 (0.31977855677171213) + | > loader_time: 0.0095 (0.009594006174260924) + + + --> TIME: 2024-10-20 06:31:30 -- STEP: 13800/16928 -- GLOBAL_STEP: 13800 + | > loss_text_ce: 0.03770546242594719 (0.04027414874254672) + | > loss_mel_ce: 3.9956343173980713 (3.8096489454525106) + | > loss: 1.008334994316101 (0.9624807737962062) + | > current_lr: 5e-06 + | > step_time: 0.3472 (0.3197775244021764) + | > loader_time: 0.009 (0.009591938205387285) + + + --> TIME: 2024-10-20 06:31:57 -- STEP: 13850/16928 -- GLOBAL_STEP: 13850 + | > loss_text_ce: 0.04269891977310181 (0.040266514683700294) + | > loss_mel_ce: 3.568566083908081 (3.8090597527810384) + | > loss: 0.9028162360191345 (0.9623315670946447) + | > current_lr: 5e-06 + | > step_time: 0.4346 (0.31978447499258017) + | > loader_time: 0.0102 (0.009591029508019187) + + + --> TIME: 2024-10-20 06:32:25 -- STEP: 13900/16928 -- GLOBAL_STEP: 13900 + | > loss_text_ce: 0.03725546598434448 (0.040259177480964865) + | > loss_mel_ce: 3.5903191566467285 (3.8084906042222495) + | > loss: 0.9068936705589294 (0.9621874456611471) + | > current_lr: 5e-06 + | > step_time: 0.2243 (0.31985870736966054) + | > loader_time: 0.0088 (0.009589863818326447) + + + --> TIME: 2024-10-20 06:32:53 -- STEP: 13950/16928 -- GLOBAL_STEP: 13950 + | > loss_text_ce: 0.03755272924900055 (0.040254912958430214) + | > loss_mel_ce: 3.650181770324707 (3.808120689973182) + | > loss: 0.921933650970459 (0.9620939009633973) + | > current_lr: 5e-06 + | > step_time: 0.2134 (0.3199126211973077) + | > loader_time: 0.0096 (0.009587862893244703) + + + --> TIME: 2024-10-20 06:33:18 -- STEP: 14000/16928 -- GLOBAL_STEP: 14000 + | > loss_text_ce: 0.03641997650265694 (0.04024817162126822) + | > loss_mel_ce: 3.88529372215271 (3.8079242165088654) + | > loss: 0.9804283976554871 (0.9620430972703868) + | > current_lr: 5e-06 + | > step_time: 0.2474 (0.3198164507831849) + | > loader_time: 0.0079 (0.009585404566356119) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_14000.pth + + --> TIME: 2024-10-20 06:33:57 -- STEP: 14050/16928 -- GLOBAL_STEP: 14050 + | > loss_text_ce: 0.041453681886196136 (0.04024358751179366) + | > loss_mel_ce: 3.7220189571380615 (3.8076999672621596) + | > loss: 0.9408681392669678 (0.9619858889180997) + | > current_lr: 5e-06 + | > step_time: 0.3373 (0.319849695175144) + | > loader_time: 0.0103 (0.009586426083303436) + + + --> TIME: 2024-10-20 06:34:25 -- STEP: 14100/16928 -- GLOBAL_STEP: 14100 + | > loss_text_ce: 0.035245832055807114 (0.04023598299579722) + | > loss_mel_ce: 3.7677628993988037 (3.8073025161323817) + | > loss: 0.9507521986961365 (0.9618846250129932) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.3198988236941347) + | > loader_time: 0.0095 (0.0095855758037973) + + + --> TIME: 2024-10-20 06:34:53 -- STEP: 14150/16928 -- GLOBAL_STEP: 14150 + | > loss_text_ce: 0.03944007307291031 (0.040230036571016245) + | > loss_mel_ce: 3.188866138458252 (3.8068492267831053) + | > loss: 0.8070765733718872 (0.9617698160793253) + | > current_lr: 5e-06 + | > step_time: 0.3102 (0.31999338114640763) + | > loader_time: 0.0086 (0.009584723344540011) + + + --> TIME: 2024-10-20 06:35:20 -- STEP: 14200/16928 -- GLOBAL_STEP: 14200 + | > loss_text_ce: 0.03636370226740837 (0.04022285187603814) + | > loss_mel_ce: 3.908026933670044 (3.8066084032663157) + | > loss: 0.9860976338386536 (0.9617078140336026) + | > current_lr: 5e-06 + | > step_time: 0.188 (0.3199937728257249) + | > loader_time: 0.0083 (0.009584881554187187) + + + --> TIME: 2024-10-20 06:35:47 -- STEP: 14250/16928 -- GLOBAL_STEP: 14250 + | > loss_text_ce: 0.04043368622660637 (0.040214849795557825) + | > loss_mel_ce: 3.2853734493255615 (3.8063979678070337) + | > loss: 0.8314517736434937 (0.961653204629296) + | > current_lr: 5e-06 + | > step_time: 0.3767 (0.3200227993747647) + | > loader_time: 0.0093 (0.009585463825025061) + + + --> TIME: 2024-10-20 06:36:15 -- STEP: 14300/16928 -- GLOBAL_STEP: 14300 + | > loss_text_ce: 0.03521298989653587 (0.040209582251971916) + | > loss_mel_ce: 3.2996554374694824 (3.8059880571098597) + | > loss: 0.8337171077728271 (0.9615494100739075) + | > current_lr: 5e-06 + | > step_time: 0.2024 (0.3200512014735831) + | > loader_time: 0.0101 (0.009584854332717153) + + + --> TIME: 2024-10-20 06:36:42 -- STEP: 14350/16928 -- GLOBAL_STEP: 14350 + | > loss_text_ce: 0.039694055914878845 (0.04020348842200593) + | > loss_mel_ce: 3.381134271621704 (3.8058751147133965) + | > loss: 0.855207085609436 (0.9615196510183689) + | > current_lr: 5e-06 + | > step_time: 0.2541 (0.32004967054838945) + | > loader_time: 0.0091 (0.009584685784183733) + + + --> TIME: 2024-10-20 06:37:08 -- STEP: 14400/16928 -- GLOBAL_STEP: 14400 + | > loss_text_ce: 0.036358918994665146 (0.04019444104814372) + | > loss_mel_ce: 4.24993896484375 (3.8059176055590314) + | > loss: 1.0715744495391846 (0.9615280118750203) + | > current_lr: 5e-06 + | > step_time: 0.1549 (0.3200178077651397) + | > loader_time: 0.0088 (0.009584204289648271) + + + --> TIME: 2024-10-20 06:37:37 -- STEP: 14450/16928 -- GLOBAL_STEP: 14450 + | > loss_text_ce: 0.03911679983139038 (0.040187700506183595) + | > loss_mel_ce: 3.7597765922546387 (3.8056904080169836) + | > loss: 0.9497233629226685 (0.9614695273509902) + | > current_lr: 5e-06 + | > step_time: 0.3794 (0.3201264257497031) + | > loader_time: 0.0104 (0.00958483210896951) + + + --> TIME: 2024-10-20 06:38:04 -- STEP: 14500/16928 -- GLOBAL_STEP: 14500 + | > loss_text_ce: 0.038081422448158264 (0.04017940308526146) + | > loss_mel_ce: 4.005511283874512 (3.805495476196552) + | > loss: 1.010898232460022 (0.961418720031607) + | > current_lr: 5e-06 + | > step_time: 0.2237 (0.3201132556652203) + | > loader_time: 0.0085 (0.00958443616998607) + + + --> TIME: 2024-10-20 06:38:31 -- STEP: 14550/16928 -- GLOBAL_STEP: 14550 + | > loss_text_ce: 0.037138231098651886 (0.04017138524576241) + | > loss_mel_ce: 3.6480932235717773 (3.8051238144386264) + | > loss: 0.9213078618049622 (0.9613238001402303) + | > current_lr: 5e-06 + | > step_time: 0.3345 (0.3201184762712199) + | > loader_time: 0.0097 (0.009584097485362054) + + + --> TIME: 2024-10-20 06:38:58 -- STEP: 14600/16928 -- GLOBAL_STEP: 14600 + | > loss_text_ce: 0.0366552472114563 (0.04016455235942063) + | > loss_mel_ce: 3.45609974861145 (3.8048127587364147) + | > loss: 0.8731887340545654 (0.9612443279811783) + | > current_lr: 5e-06 + | > step_time: 0.3068 (0.3201350935354627) + | > loader_time: 0.0108 (0.009582995845846938) + + + --> TIME: 2024-10-20 06:39:26 -- STEP: 14650/16928 -- GLOBAL_STEP: 14650 + | > loss_text_ce: 0.033538639545440674 (0.04015731308710949) + | > loss_mel_ce: 3.572171926498413 (3.804479358643802) + | > loss: 0.9014276266098022 (0.9611591681441352) + | > current_lr: 5e-06 + | > step_time: 0.3583 (0.3201545736325888) + | > loader_time: 0.0097 (0.00958251461640967) + + + --> TIME: 2024-10-20 06:39:51 -- STEP: 14700/16928 -- GLOBAL_STEP: 14700 + | > loss_text_ce: 0.037899840623140335 (0.04015075947215046) + | > loss_mel_ce: 4.179494857788086 (3.804244122505188) + | > loss: 1.0543487071990967 (0.9610987207070503) + | > current_lr: 5e-06 + | > step_time: 0.2735 (0.32009176734353434) + | > loader_time: 0.0109 (0.009581905335796127) + + + --> TIME: 2024-10-20 06:40:19 -- STEP: 14750/16928 -- GLOBAL_STEP: 14750 + | > loss_text_ce: 0.037646885961294174 (0.040143162984337726) + | > loss_mel_ce: 3.2574026584625244 (3.8039594497519023) + | > loss: 0.8237623572349548 (0.9610256533905613) + | > current_lr: 5e-06 + | > step_time: 0.4269 (0.3201317430108283) + | > loader_time: 0.0086 (0.009581290115744384) + + + --> TIME: 2024-10-20 06:40:46 -- STEP: 14800/16928 -- GLOBAL_STEP: 14800 + | > loss_text_ce: 0.041491616517305374 (0.04013700372199641) + | > loss_mel_ce: 3.7748286724090576 (3.8036014921761847) + | > loss: 0.9540800452232361 (0.960934624184628) + | > current_lr: 5e-06 + | > step_time: 0.1825 (0.32012509260628697) + | > loader_time: 0.009 (0.009580775128828517) + + + --> TIME: 2024-10-20 06:41:11 -- STEP: 14850/16928 -- GLOBAL_STEP: 14850 + | > loss_text_ce: 0.036202818155288696 (0.0401286916837396) + | > loss_mel_ce: 3.4912683963775635 (3.803082777000838) + | > loss: 0.8818678259849548 (0.9608028673924984) + | > current_lr: 5e-06 + | > step_time: 0.2766 (0.32008381018333587) + | > loader_time: 0.0094 (0.009580986604144682) + + + --> TIME: 2024-10-20 06:41:37 -- STEP: 14900/16928 -- GLOBAL_STEP: 14900 + | > loss_text_ce: 0.03804031386971474 (0.04012005157381184) + | > loss_mel_ce: 4.011920928955078 (3.802863863618582) + | > loss: 1.0124902725219727 (0.9607459790194597) + | > current_lr: 5e-06 + | > step_time: 0.2882 (0.3200385374350838) + | > loader_time: 0.0084 (0.009579893390604317) + + + --> TIME: 2024-10-20 06:42:04 -- STEP: 14950/16928 -- GLOBAL_STEP: 14950 + | > loss_text_ce: 0.03599521517753601 (0.04011243796779647) + | > loss_mel_ce: 3.3476154804229736 (3.8028226501088476) + | > loss: 0.845902681350708 (0.9607337722531133) + | > current_lr: 5e-06 + | > step_time: 0.2543 (0.32003694531112237) + | > loader_time: 0.009 (0.009579000393283809) + + + --> TIME: 2024-10-20 06:42:33 -- STEP: 15000/16928 -- GLOBAL_STEP: 15000 + | > loss_text_ce: 0.03828585520386696 (0.040105076319351673) + | > loss_mel_ce: 3.667917490005493 (3.8025345842997234) + | > loss: 0.9265508651733398 (0.960659915383657) + | > current_lr: 5e-06 + | > step_time: 0.2404 (0.32010155574480714) + | > loader_time: 0.01 (0.009578665717442834) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_15000.pth + + --> TIME: 2024-10-20 06:43:09 -- STEP: 15050/16928 -- GLOBAL_STEP: 15050 + | > loss_text_ce: 0.04206215217709541 (0.04009856628483007) + | > loss_mel_ce: 3.8085076808929443 (3.80213823323234) + | > loss: 0.9626424312591553 (0.9605592001078536) + | > current_lr: 5e-06 + | > step_time: 0.2085 (0.3200250639075855) + | > loader_time: 0.0088 (0.009578771543661229) + + + --> TIME: 2024-10-20 06:43:36 -- STEP: 15100/16928 -- GLOBAL_STEP: 15100 + | > loss_text_ce: 0.035359010100364685 (0.04009183465509312) + | > loss_mel_ce: 3.526052474975586 (3.8018361451452143) + | > loss: 0.8903528451919556 (0.9604819951704798) + | > current_lr: 5e-06 + | > step_time: 0.2447 (0.3200185389550319) + | > loader_time: 0.0086 (0.009577454146959928) + + + --> TIME: 2024-10-20 06:44:01 -- STEP: 15150/16928 -- GLOBAL_STEP: 15150 + | > loss_text_ce: 0.035303521901369095 (0.04008823670950943) + | > loss_mel_ce: 3.7194063663482666 (3.8013479961577814) + | > loss: 0.9386774897575378 (0.9603590584391417) + | > current_lr: 5e-06 + | > step_time: 0.3093 (0.31996581146819386) + | > loader_time: 0.0091 (0.009576558075328869) + + + --> TIME: 2024-10-20 06:44:30 -- STEP: 15200/16928 -- GLOBAL_STEP: 15200 + | > loss_text_ce: 0.03921296447515488 (0.04008259220110044) + | > loss_mel_ce: 3.7789032459259033 (3.8010143418374813) + | > loss: 0.9545290470123291 (0.9602742337278632) + | > current_lr: 5e-06 + | > step_time: 0.2739 (0.32004448184841583) + | > loader_time: 0.0102 (0.009575998390975756) + + + --> TIME: 2024-10-20 06:44:58 -- STEP: 15250/16928 -- GLOBAL_STEP: 15250 + | > loss_text_ce: 0.04069148749113083 (0.040077812213878045) + | > loss_mel_ce: 4.1000752449035645 (3.800563268083041) + | > loss: 1.0351916551589966 (0.9601602703000681) + | > current_lr: 5e-06 + | > step_time: 0.338 (0.32008379393718306) + | > loader_time: 0.0083 (0.009575092456379879) + + + --> TIME: 2024-10-20 06:45:26 -- STEP: 15300/16928 -- GLOBAL_STEP: 15300 + | > loss_text_ce: 0.038064129650592804 (0.04006995598036857) + | > loss_mel_ce: 3.354496717453003 (3.8002713060846514) + | > loss: 0.8481402397155762 (0.9600853157355118) + | > current_lr: 5e-06 + | > step_time: 0.3572 (0.32014785461176476) + | > loader_time: 0.0099 (0.009575194299610615) + + + --> TIME: 2024-10-20 06:45:53 -- STEP: 15350/16928 -- GLOBAL_STEP: 15350 + | > loss_text_ce: 0.036727260798215866 (0.040062416641205335) + | > loss_mel_ce: 3.8306446075439453 (3.800025053319403) + | > loss: 0.9668429493904114 (0.960021867709362) + | > current_lr: 5e-06 + | > step_time: 0.4461 (0.320164632750645) + | > loader_time: 0.0115 (0.009575173054921903) + + + --> TIME: 2024-10-20 06:46:19 -- STEP: 15400/16928 -- GLOBAL_STEP: 15400 + | > loss_text_ce: 0.040985431522130966 (0.04005625702247205) + | > loss_mel_ce: 3.6529011726379395 (3.7998613917982422) + | > loss: 0.9234716296195984 (0.9599794124279706) + | > current_lr: 5e-06 + | > step_time: 0.2112 (0.3200991594946232) + | > loader_time: 0.009 (0.00957481028197648) + + + --> TIME: 2024-10-20 06:46:45 -- STEP: 15450/16928 -- GLOBAL_STEP: 15450 + | > loss_text_ce: 0.03534393012523651 (0.04004926513771976) + | > loss_mel_ce: 3.517855405807495 (3.7995794343176783) + | > loss: 0.888299822807312 (0.9599071750980367) + | > current_lr: 5e-06 + | > step_time: 0.3036 (0.32007439712104696) + | > loader_time: 0.0086 (0.009573992756963938) + + + --> TIME: 2024-10-20 06:47:10 -- STEP: 15500/16928 -- GLOBAL_STEP: 15500 + | > loss_text_ce: 0.037736449390649796 (0.04004182614554309) + | > loss_mel_ce: 3.390737771987915 (3.7989784476218684) + | > loss: 0.857118546962738 (0.9597550686559372) + | > current_lr: 5e-06 + | > step_time: 0.2455 (0.3200114228187071) + | > loader_time: 0.0096 (0.009573890701417004) + + + --> TIME: 2024-10-20 06:47:39 -- STEP: 15550/16928 -- GLOBAL_STEP: 15550 + | > loss_text_ce: 0.036094240844249725 (0.0400350965760719) + | > loss_mel_ce: 3.536532163619995 (3.7985470758297053) + | > loss: 0.8931565880775452 (0.9596455433207696) + | > current_lr: 5e-06 + | > step_time: 0.2955 (0.32008548672176274) + | > loader_time: 0.0088 (0.009573753761708548) + + + --> TIME: 2024-10-20 06:48:06 -- STEP: 15600/16928 -- GLOBAL_STEP: 15600 + | > loss_text_ce: 0.03378576040267944 (0.04002724187424729) + | > loss_mel_ce: 3.751279354095459 (3.7983557891845705) + | > loss: 0.9462662935256958 (0.9595957579673867) + | > current_lr: 5e-06 + | > step_time: 0.3196 (0.32009750736065423) + | > loader_time: 0.0097 (0.009574231413694533) + + + --> TIME: 2024-10-20 06:48:31 -- STEP: 15650/16928 -- GLOBAL_STEP: 15650 + | > loss_text_ce: 0.03901934251189232 (0.04001985162615574) + | > loss_mel_ce: 3.8250205516815186 (3.7980736772092385) + | > loss: 0.9660099744796753 (0.9595233824154061) + | > current_lr: 5e-06 + | > step_time: 0.4415 (0.3200416073326869) + | > loader_time: 0.0109 (0.009573290172857222) + + + --> TIME: 2024-10-20 06:48:58 -- STEP: 15700/16928 -- GLOBAL_STEP: 15700 + | > loss_text_ce: 0.03594905510544777 (0.04001221646928472) + | > loss_mel_ce: 4.290029048919678 (3.797793221002931) + | > loss: 1.0814945697784424 (0.9594513595749621) + | > current_lr: 5e-06 + | > step_time: 0.2897 (0.32002919564581245) + | > loader_time: 0.0096 (0.009572567104533983) + + + --> TIME: 2024-10-20 06:49:25 -- STEP: 15750/16928 -- GLOBAL_STEP: 15750 + | > loss_text_ce: 0.03484335541725159 (0.040005267397278714) + | > loss_mel_ce: 3.6109366416931152 (3.7976084855548917) + | > loss: 0.9114450216293335 (0.9594034384394451) + | > current_lr: 5e-06 + | > step_time: 0.3945 (0.32004280300745913) + | > loader_time: 0.0094 (0.009572156860714871) + + + --> TIME: 2024-10-20 06:49:52 -- STEP: 15800/16928 -- GLOBAL_STEP: 15800 + | > loss_text_ce: 0.04126174747943878 (0.0399993370023048) + | > loss_mel_ce: 4.276076316833496 (3.7972712424435193) + | > loss: 1.0793344974517822 (0.9593176450684103) + | > current_lr: 5e-06 + | > step_time: 0.4176 (0.32003407209734397) + | > loader_time: 0.0098 (0.009571479709842541) + + + --> TIME: 2024-10-20 06:50:19 -- STEP: 15850/16928 -- GLOBAL_STEP: 15850 + | > loss_text_ce: 0.034573961049318314 (0.039991862580697125) + | > loss_mel_ce: 4.871270656585693 (3.797073474035654) + | > loss: 1.2264611721038818 (0.9592663343697319) + | > current_lr: 5e-06 + | > step_time: 0.3859 (0.32006510608203803) + | > loader_time: 0.0111 (0.009570794090487608) + + + --> TIME: 2024-10-20 06:50:45 -- STEP: 15900/16928 -- GLOBAL_STEP: 15900 + | > loss_text_ce: 0.031156646087765694 (0.03998388554748582) + | > loss_mel_ce: 4.134383678436279 (3.796535245097658) + | > loss: 1.0413850545883179 (0.9591297828888745) + | > current_lr: 5e-06 + | > step_time: 0.3007 (0.32002869328612826) + | > loader_time: 0.0092 (0.00956980592799637) + + + --> TIME: 2024-10-20 06:51:14 -- STEP: 15950/16928 -- GLOBAL_STEP: 15950 + | > loss_text_ce: 0.03496754169464111 (0.039977288454607435) + | > loss_mel_ce: 4.061573028564453 (3.796341910182869) + | > loss: 1.0241351127624512 (0.9590797998875289) + | > current_lr: 5e-06 + | > step_time: 0.5097 (0.32013091561188917) + | > loader_time: 0.0094 (0.00956917078143751) + + + --> TIME: 2024-10-20 06:51:40 -- STEP: 16000/16928 -- GLOBAL_STEP: 16000 + | > loss_text_ce: 0.034305159002542496 (0.03996914316492609) + | > loss_mel_ce: 3.829882860183716 (3.795941762432456) + | > loss: 0.9660469889640808 (0.9589777266420426) + | > current_lr: 5e-06 + | > step_time: 0.1729 (0.3200892716497185) + | > loader_time: 0.0087 (0.009568334057927135) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_16000.pth + + --> TIME: 2024-10-20 06:52:22 -- STEP: 16050/16928 -- GLOBAL_STEP: 16050 + | > loss_text_ce: 0.034586627036333084 (0.0399616212607868) + | > loss_mel_ce: 3.2368123531341553 (3.7955827218052756) + | > loss: 0.8178497552871704 (0.9588860860086307) + | > current_lr: 5e-06 + | > step_time: 0.2262 (0.3200998472647506) + | > loader_time: 0.0093 (0.009569311052839336) + + + --> TIME: 2024-10-20 06:52:47 -- STEP: 16100/16928 -- GLOBAL_STEP: 16100 + | > loss_text_ce: 0.03432685509324074 (0.03995444620745419) + | > loss_mel_ce: 3.4777333736419678 (3.7951830831225615) + | > loss: 0.8780150413513184 (0.9587843825617194) + | > current_lr: 5e-06 + | > step_time: 0.2573 (0.32003897080510324) + | > loader_time: 0.0103 (0.00956748541837894) + + + --> TIME: 2024-10-20 06:53:13 -- STEP: 16150/16928 -- GLOBAL_STEP: 16150 + | > loss_text_ce: 0.036536432802677155 (0.039949406700168305) + | > loss_mel_ce: 3.6872501373291016 (3.7950436629304205) + | > loss: 0.9309466481208801 (0.9587482676343655) + | > current_lr: 5e-06 + | > step_time: 0.3565 (0.3199988830200292) + | > loader_time: 0.0084 (0.009565868628652476) + + + --> TIME: 2024-10-20 06:53:42 -- STEP: 16200/16928 -- GLOBAL_STEP: 16200 + | > loss_text_ce: 0.03830550238490105 (0.039942906182374185) + | > loss_mel_ce: 3.838015079498291 (3.7948225157790714) + | > loss: 0.9690801501274109 (0.9586913557148277) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.3200894998326716) + | > loader_time: 0.0089 (0.009565305121151021) + + + --> TIME: 2024-10-20 06:54:09 -- STEP: 16250/16928 -- GLOBAL_STEP: 16250 + | > loss_text_ce: 0.041046515107154846 (0.039937608415461394) + | > loss_mel_ce: 3.5580673217773438 (3.794450132590074) + | > loss: 0.8997784852981567 (0.9585969354812918) + | > current_lr: 5e-06 + | > step_time: 0.2576 (0.3200533923075752) + | > loader_time: 0.0083 (0.009563389396667484) + + + --> TIME: 2024-10-20 06:54:36 -- STEP: 16300/16928 -- GLOBAL_STEP: 16300 + | > loss_text_ce: 0.04004145413637161 (0.03993247528737571) + | > loss_mel_ce: 3.712826728820801 (3.794061024744818) + | > loss: 0.938217043876648 (0.9584983752402794) + | > current_lr: 5e-06 + | > step_time: 0.2076 (0.32008534977041164) + | > loader_time: 0.0102 (0.00956239997243589) + + + --> TIME: 2024-10-20 06:55:04 -- STEP: 16350/16928 -- GLOBAL_STEP: 16350 + | > loss_text_ce: 0.037455521523952484 (0.0399274019132558) + | > loss_mel_ce: 4.175843715667725 (3.7939503972318924) + | > loss: 1.053324818611145 (0.9584694500180926) + | > current_lr: 5e-06 + | > step_time: 0.3742 (0.32012488404545236) + | > loader_time: 0.0095 (0.009560877175870659) + + + --> TIME: 2024-10-20 06:55:30 -- STEP: 16400/16928 -- GLOBAL_STEP: 16400 + | > loss_text_ce: 0.03791749104857445 (0.03991921769557283) + | > loss_mel_ce: 4.0298051834106445 (3.793543517589569) + | > loss: 1.0169306993484497 (0.9583656840571545) + | > current_lr: 5e-06 + | > step_time: 0.4206 (0.3201276537703307) + | > loader_time: 0.0109 (0.00955958468158071) + + + --> TIME: 2024-10-20 06:55:57 -- STEP: 16450/16928 -- GLOBAL_STEP: 16450 + | > loss_text_ce: 0.03722470626235008 (0.03991201875136876) + | > loss_mel_ce: 3.4062116146087646 (3.7931478607038596) + | > loss: 0.8608590960502625 (0.9582649701018466) + | > current_lr: 5e-06 + | > step_time: 0.4286 (0.3201145305024819) + | > loader_time: 0.009 (0.009558878669622824) + + + --> TIME: 2024-10-20 06:56:25 -- STEP: 16500/16928 -- GLOBAL_STEP: 16500 + | > loss_text_ce: 0.03993230313062668 (0.039905156754973144) + | > loss_mel_ce: 4.029303073883057 (3.79300255687309) + | > loss: 1.0173088312149048 (0.9582269286386895) + | > current_lr: 5e-06 + | > step_time: 0.1545 (0.32015917801134536) + | > loader_time: 0.0093 (0.009558650984908598) + + + --> TIME: 2024-10-20 06:56:51 -- STEP: 16550/16928 -- GLOBAL_STEP: 16550 + | > loss_text_ce: 0.038554202765226364 (0.039898048432774474) + | > loss_mel_ce: 3.6409757137298584 (3.7929433458161137) + | > loss: 0.9198824763298035 (0.9582103487948278) + | > current_lr: 5e-06 + | > step_time: 0.3607 (0.3201443145066232) + | > loader_time: 0.0092 (0.009558588010666955) + + + --> TIME: 2024-10-20 06:57:17 -- STEP: 16600/16928 -- GLOBAL_STEP: 16600 + | > loss_text_ce: 0.03485589101910591 (0.03989242637967567) + | > loss_mel_ce: 3.2292416095733643 (3.792564956653549) + | > loss: 0.8160243630409241 (0.9581143459964949) + | > current_lr: 5e-06 + | > step_time: 0.2043 (0.32010229464036893) + | > loader_time: 0.0086 (0.009558017110250085) + + + --> TIME: 2024-10-20 06:57:44 -- STEP: 16650/16928 -- GLOBAL_STEP: 16650 + | > loss_text_ce: 0.03314393386244774 (0.03988780103746913) + | > loss_mel_ce: 3.7884323596954346 (3.792322832714688) + | > loss: 0.9553940892219543 (0.9580526586767434) + | > current_lr: 5e-06 + | > step_time: 0.3398 (0.3200927074392281) + | > loader_time: 0.0091 (0.009557602212235736) + + + --> TIME: 2024-10-20 06:58:10 -- STEP: 16700/16928 -- GLOBAL_STEP: 16700 + | > loss_text_ce: 0.037483006715774536 (0.03988168594131826) + | > loss_mel_ce: 3.4549622535705566 (3.792112668103086) + | > loss: 0.8731113076210022 (0.9579985887418966) + | > current_lr: 5e-06 + | > step_time: 0.2761 (0.32006945378766116) + | > loader_time: 0.0093 (0.009557017623307464) + + + --> TIME: 2024-10-20 06:58:37 -- STEP: 16750/16928 -- GLOBAL_STEP: 16750 + | > loss_text_ce: 0.03687673062086105 (0.0398773656250602) + | > loss_mel_ce: 3.6248538494110107 (3.7918553459395223) + | > loss: 0.9154326319694519 (0.9579331781188056) + | > current_lr: 5e-06 + | > step_time: 0.4003 (0.3200588965060108) + | > loader_time: 0.0104 (0.009556010175107134) + + + --> TIME: 2024-10-20 06:59:04 -- STEP: 16800/16928 -- GLOBAL_STEP: 16800 + | > loss_text_ce: 0.03381773456931114 (0.03987104510589092) + | > loss_mel_ce: 4.124736309051514 (3.7917560983555654) + | > loss: 1.0396385192871094 (0.95790678607566) + | > current_lr: 5e-06 + | > step_time: 0.2179 (0.32007522790204945) + | > loader_time: 0.0088 (0.009555322244053803) + + + --> TIME: 2024-10-20 06:59:29 -- STEP: 16850/16928 -- GLOBAL_STEP: 16850 + | > loss_text_ce: 0.036686450242996216 (0.03986640822739282) + | > loss_mel_ce: 3.57902193069458 (3.791372978467969) + | > loss: 0.9039270877838135 (0.9578098468787591) + | > current_lr: 5e-06 + | > step_time: 0.4302 (0.320022188718666) + | > loader_time: 0.0103 (0.00955498572273142) + + + --> TIME: 2024-10-20 06:59:56 -- STEP: 16900/16928 -- GLOBAL_STEP: 16900 + | > loss_text_ce: 0.03686889633536339 (0.039859327077755274) + | > loss_mel_ce: 3.7410473823547363 (3.7910353712076263) + | > loss: 0.9444790482521057 (0.957723674770643) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.32000189317048683) + | > loader_time: 0.0083 (0.009554282476210743) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.3729819632288238 (+0) + | > avg_loss_text_ce: 0.03591250202326634 (+0) + | > avg_loss_mel_ce: 3.519801580372141 (+0) + | > avg_loss: 3.5557140862763825 (+0) + + > BEST MODEL : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/best_model_16928.pth + + > EPOCH: 1/5 + --> /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b + + > TRAINING (2024-10-20 07:10:59)  + + --> TIME: 2024-10-20 07:11:12 -- STEP: 22/16928 -- GLOBAL_STEP: 16950 + | > loss_text_ce: 0.03675822168588638 (0.03739859654822133) + | > loss_mel_ce: 3.452984571456909 (3.6562025113539263) + | > loss: 0.8724356889724731 (0.9234002747318961) + | > current_lr: 5e-06 + | > step_time: 0.2535 (0.29850584810430353) + | > loader_time: 0.0086 (0.01021508737043901) + + + --> TIME: 2024-10-20 07:11:39 -- STEP: 72/16928 -- GLOBAL_STEP: 17000 + | > loss_text_ce: 0.036452244967222214 (0.03734382583449283) + | > loss_mel_ce: 3.671546697616577 (3.674231082201004) + | > loss: 0.9269997477531433 (0.927893722222911) + | > current_lr: 5e-06 + | > step_time: 0.2716 (0.31617498728964066) + | > loader_time: 0.0091 (0.010202060143152876) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_17000.pth + + --> TIME: 2024-10-20 07:12:15 -- STEP: 122/16928 -- GLOBAL_STEP: 17050 + | > loss_text_ce: 0.03691408410668373 (0.03745022697038338) + | > loss_mel_ce: 3.7527027130126953 (3.654392830661086) + | > loss: 0.9474042057991028 (0.9229607611406044) + | > current_lr: 5e-06 + | > step_time: 0.3074 (0.3112581049809691) + | > loader_time: 0.0102 (0.010240984744712954) + + + --> TIME: 2024-10-20 07:12:43 -- STEP: 172/16928 -- GLOBAL_STEP: 17100 + | > loss_text_ce: 0.038083259016275406 (0.037400224809209986) + | > loss_mel_ce: 3.9013075828552246 (3.6610013870305793) + | > loss: 0.9848477244377136 (0.9246003991642662) + | > current_lr: 5e-06 + | > step_time: 0.3207 (0.31696051359176647) + | > loader_time: 0.0111 (0.010028841883637188) + + + --> TIME: 2024-10-20 07:13:10 -- STEP: 222/16928 -- GLOBAL_STEP: 17150 + | > loss_text_ce: 0.036833081394433975 (0.037416697001537776) + | > loss_mel_ce: 3.2534310817718506 (3.676165070619669) + | > loss: 0.822566032409668 (0.9283954391071387) + | > current_lr: 5e-06 + | > step_time: 0.2546 (0.3190773897342855) + | > loader_time: 0.01 (0.009874252585677418) + + + --> TIME: 2024-10-20 07:13:38 -- STEP: 272/16928 -- GLOBAL_STEP: 17200 + | > loss_text_ce: 0.04280570521950722 (0.03740907669998707) + | > loss_mel_ce: 3.746199369430542 (3.6779755301335277) + | > loss: 0.9472512602806091 (0.9288461484453256) + | > current_lr: 5e-06 + | > step_time: 0.4069 (0.32022862925248996) + | > loader_time: 0.0097 (0.009760373655487512) + + + --> TIME: 2024-10-20 07:14:04 -- STEP: 322/16928 -- GLOBAL_STEP: 17250 + | > loss_text_ce: 0.04447324201464653 (0.037493917477889854) + | > loss_mel_ce: 4.030598163604736 (3.6965966691141543) + | > loss: 1.0187678337097168 (0.9335226433247513) + | > current_lr: 5e-06 + | > step_time: 0.2279 (0.31957726300873374) + | > loader_time: 0.0108 (0.009711445488544725) + + + --> TIME: 2024-10-20 07:14:31 -- STEP: 372/16928 -- GLOBAL_STEP: 17300 + | > loss_text_ce: 0.03542843088507652 (0.037506315126133886) + | > loss_mel_ce: 3.2882797718048096 (3.6869859362161286) + | > loss: 0.8309270739555359 (0.9311230599239309) + | > current_lr: 5e-06 + | > step_time: 0.2498 (0.3189556694799855) + | > loader_time: 0.0096 (0.009708617964098531) + + + --> TIME: 2024-10-20 07:14:58 -- STEP: 422/16928 -- GLOBAL_STEP: 17350 + | > loss_text_ce: 0.038536906242370605 (0.037535729681216726) + | > loss_mel_ce: 3.6089365482330322 (3.6927056041374025) + | > loss: 0.9118683338165283 (0.9325603307423433) + | > current_lr: 5e-06 + | > step_time: 0.3096 (0.31970775635886534) + | > loader_time: 0.009 (0.009666469424821753) + + + --> TIME: 2024-10-20 07:15:25 -- STEP: 472/16928 -- GLOBAL_STEP: 17400 + | > loss_text_ce: 0.038467247039079666 (0.03761161370518603) + | > loss_mel_ce: 3.51419997215271 (3.688925001075712) + | > loss: 0.8881667852401733 (0.9316341515819905) + | > current_lr: 5e-06 + | > step_time: 0.3018 (0.32111001115734294) + | > loader_time: 0.0092 (0.00966558921135078) + + + --> TIME: 2024-10-20 07:15:53 -- STEP: 522/16928 -- GLOBAL_STEP: 17450 + | > loss_text_ce: 0.03452267870306969 (0.03756615308994764) + | > loss_mel_ce: 3.450314521789551 (3.6907976477538944) + | > loss: 0.8712093234062195 (0.9320909488246816) + | > current_lr: 5e-06 + | > step_time: 0.2518 (0.32154314591053346) + | > loader_time: 0.0088 (0.009642717938770277) + + + --> TIME: 2024-10-20 07:16:20 -- STEP: 572/16928 -- GLOBAL_STEP: 17500 + | > loss_text_ce: 0.036386094987392426 (0.03754939616924817) + | > loss_mel_ce: 3.4244046211242676 (3.6958853719117775) + | > loss: 0.8651976585388184 (0.9333586903301986) + | > current_lr: 5e-06 + | > step_time: 0.2384 (0.3218447274261422) + | > loader_time: 0.0102 (0.009620514366176582) + + + --> TIME: 2024-10-20 07:16:47 -- STEP: 622/16928 -- GLOBAL_STEP: 17550 + | > loss_text_ce: 0.036181431263685226 (0.03751242929213113) + | > loss_mel_ce: 3.428314208984375 (3.6934790047801975) + | > loss: 0.8661239147186279 (0.9327478570761788) + | > current_lr: 5e-06 + | > step_time: 0.3095 (0.3219576525918158) + | > loader_time: 0.0103 (0.009610248531942592) + + + --> TIME: 2024-10-20 07:17:13 -- STEP: 672/16928 -- GLOBAL_STEP: 17600 + | > loss_text_ce: 0.03742048516869545 (0.03750481207611132) + | > loss_mel_ce: 3.8715789318084717 (3.6979221598733054) + | > loss: 0.9772498607635498 (0.9338567420130685) + | > current_lr: 5e-06 + | > step_time: 0.1485 (0.32163606867903755) + | > loader_time: 0.0082 (0.009602228090876632) + + + --> TIME: 2024-10-20 07:17:40 -- STEP: 722/16928 -- GLOBAL_STEP: 17650 + | > loss_text_ce: 0.04013162851333618 (0.03751926377952271) + | > loss_mel_ce: 3.547086000442505 (3.6990101043537393) + | > loss: 0.8968043923377991 (0.9341323410374966) + | > current_lr: 5e-06 + | > step_time: 0.3369 (0.3211256354799561) + | > loader_time: 0.0092 (0.009582075385836039) + + + --> TIME: 2024-10-20 07:18:06 -- STEP: 772/16928 -- GLOBAL_STEP: 17700 + | > loss_text_ce: 0.043756891041994095 (0.03753243435965614) + | > loss_mel_ce: 3.0384397506713867 (3.695293515457392) + | > loss: 0.7705491781234741 (0.9332064865012243) + | > current_lr: 5e-06 + | > step_time: 0.2252 (0.32074935386835607) + | > loader_time: 0.0096 (0.009563647094785875) + + + --> TIME: 2024-10-20 07:18:34 -- STEP: 822/16928 -- GLOBAL_STEP: 17750 + | > loss_text_ce: 0.037330497056245804 (0.037511715098961736) + | > loss_mel_ce: 3.369307041168213 (3.688508982205914) + | > loss: 0.85165935754776 (0.9315051732272127) + | > current_lr: 5e-06 + | > step_time: 0.4423 (0.3212700516638093) + | > loader_time: 0.0093 (0.009549334101433318) + + + --> TIME: 2024-10-20 07:19:00 -- STEP: 872/16928 -- GLOBAL_STEP: 17800 + | > loss_text_ce: 0.03909963369369507 (0.03749592694958007) + | > loss_mel_ce: 3.865056037902832 (3.6893748312915147) + | > loss: 0.976038932800293 (0.9317176886107943) + | > current_lr: 5e-06 + | > step_time: 0.3046 (0.3203830374490229) + | > loader_time: 0.0093 (0.009529190325955747) + + + --> TIME: 2024-10-20 07:19:28 -- STEP: 922/16928 -- GLOBAL_STEP: 17850 + | > loss_text_ce: 0.04207612946629524 (0.0375004276762557) + | > loss_mel_ce: 3.888965606689453 (3.6903108115310017) + | > loss: 0.9827604293823242 (0.9319528088523095) + | > current_lr: 5e-06 + | > step_time: 0.3358 (0.3212205075907344) + | > loader_time: 0.0092 (0.009518708686249388) + + + --> TIME: 2024-10-20 07:19:54 -- STEP: 972/16928 -- GLOBAL_STEP: 17900 + | > loss_text_ce: 0.03557157889008522 (0.03751864014461699) + | > loss_mel_ce: 4.101449966430664 (3.690693038481254) + | > loss: 1.0342553853988647 (0.932052918483691) + | > current_lr: 5e-06 + | > step_time: 0.2249 (0.3206167873531701) + | > loader_time: 0.0093 (0.009514791484722888) + + + --> TIME: 2024-10-20 07:20:20 -- STEP: 1022/16928 -- GLOBAL_STEP: 17950 + | > loss_text_ce: 0.043014321476221085 (0.03751053224425261) + | > loss_mel_ce: 3.7769250869750977 (3.6869796572832687) + | > loss: 0.9549848437309265 (0.9311225461516595) + | > current_lr: 5e-06 + | > step_time: 0.3037 (0.3200896147879135) + | > loader_time: 0.01 (0.009510107702938324) + + + --> TIME: 2024-10-20 07:20:45 -- STEP: 1072/16928 -- GLOBAL_STEP: 18000 + | > loss_text_ce: 0.03406977653503418 (0.037516223377923485) + | > loss_mel_ce: 3.8904976844787598 (3.6861924193243487) + | > loss: 0.9811418652534485 (0.9309271595704911) + | > current_lr: 5e-06 + | > step_time: 0.2033 (0.3192932659120701) + | > loader_time: 0.009 (0.00951059162616731) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_18000.pth + + --> TIME: 2024-10-20 07:21:24 -- STEP: 1122/16928 -- GLOBAL_STEP: 18050 + | > loss_text_ce: 0.03577863425016403 (0.03752829389457399) + | > loss_mel_ce: 3.412444591522217 (3.687841155312278) + | > loss: 0.862055778503418 (0.9313423612326014) + | > current_lr: 5e-06 + | > step_time: 0.3973 (0.3197061040184712) + | > loader_time: 0.009 (0.009518140777547103) + + + --> TIME: 2024-10-20 07:21:51 -- STEP: 1172/16928 -- GLOBAL_STEP: 18100 + | > loss_text_ce: 0.03590509295463562 (0.037506762354828596) + | > loss_mel_ce: 3.281682014465332 (3.684556537113906) + | > loss: 0.8293967843055725 (0.9305158237650127) + | > current_lr: 5e-06 + | > step_time: 0.2892 (0.3199828145040179) + | > loader_time: 0.0087 (0.009507911604012258) + + + --> TIME: 2024-10-20 07:22:17 -- STEP: 1222/16928 -- GLOBAL_STEP: 18150 + | > loss_text_ce: 0.03834538906812668 (0.03752205617811207) + | > loss_mel_ce: 3.5625362396240234 (3.6846468138421615) + | > loss: 0.9002203941345215 (0.9305422164121745) + | > current_lr: 5e-06 + | > step_time: 0.4451 (0.3197492101578781) + | > loader_time: 0.0097 (0.009513009589549584) + + + --> TIME: 2024-10-20 07:22:45 -- STEP: 1272/16928 -- GLOBAL_STEP: 18200 + | > loss_text_ce: 0.03976166993379593 (0.037516387153135725) + | > loss_mel_ce: 3.564577341079712 (3.6845233663448) + | > loss: 0.9010847806930542 (0.9305099374468223) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.32032219287734337) + | > loader_time: 0.0088 (0.00949460530431017) + + + --> TIME: 2024-10-20 07:23:10 -- STEP: 1322/16928 -- GLOBAL_STEP: 18250 + | > loss_text_ce: 0.03862503916025162 (0.037515578850675475) + | > loss_mel_ce: 3.4553253650665283 (3.6823258856240995) + | > loss: 0.8734875917434692 (0.9299603651493293) + | > current_lr: 5e-06 + | > step_time: 0.3585 (0.3197119939345272) + | > loader_time: 0.0099 (0.009489521857649784) + + + --> TIME: 2024-10-20 07:23:37 -- STEP: 1372/16928 -- GLOBAL_STEP: 18300 + | > loss_text_ce: 0.03687033802270889 (0.037518083875450366) + | > loss_mel_ce: 3.6177051067352295 (3.6821736264159313) + | > loss: 0.9136438369750977 (0.9299229265301987) + | > current_lr: 5e-06 + | > step_time: 0.2552 (0.3198191829395015) + | > loader_time: 0.011 (0.009476173549629876) + + + --> TIME: 2024-10-20 07:24:03 -- STEP: 1422/16928 -- GLOBAL_STEP: 18350 + | > loss_text_ce: 0.03734259679913521 (0.03750961603997627) + | > loss_mel_ce: 3.4419591426849365 (3.682619272069757) + | > loss: 0.8698254227638245 (0.9300322210319268) + | > current_lr: 5e-06 + | > step_time: 0.3763 (0.31943836980060525) + | > loader_time: 0.0101 (0.00947751978781657) + + + --> TIME: 2024-10-20 07:24:30 -- STEP: 1472/16928 -- GLOBAL_STEP: 18400 + | > loss_text_ce: 0.04060712829232216 (0.0374945713429833) + | > loss_mel_ce: 3.831087827682495 (3.683944152587134) + | > loss: 0.9679237604141235 (0.9303596800841069) + | > current_lr: 5e-06 + | > step_time: 0.2863 (0.31944052810254303) + | > loader_time: 0.0099 (0.009472092046685854) + + + --> TIME: 2024-10-20 07:24:57 -- STEP: 1522/16928 -- GLOBAL_STEP: 18450 + | > loss_text_ce: 0.03584347292780876 (0.0374753415907651) + | > loss_mel_ce: 3.666430950164795 (3.6846882899080096) + | > loss: 0.9255685806274414 (0.9305409069678475) + | > current_lr: 5e-06 + | > step_time: 0.358 (0.3194807203934478) + | > loader_time: 0.0099 (0.00947179180250533) + + + --> TIME: 2024-10-20 07:25:22 -- STEP: 1572/16928 -- GLOBAL_STEP: 18500 + | > loss_text_ce: 0.035124052315950394 (0.037470038006266984) + | > loss_mel_ce: 3.608445644378662 (3.6823317140113305) + | > loss: 0.9108924269676208 (0.9299504371696454) + | > current_lr: 5e-06 + | > step_time: 0.1828 (0.3187184075969476) + | > loader_time: 0.0091 (0.009466154126417256) + + + --> TIME: 2024-10-20 07:25:50 -- STEP: 1622/16928 -- GLOBAL_STEP: 18550 + | > loss_text_ce: 0.0362551175057888 (0.037466754808415555) + | > loss_mel_ce: 3.4333090782165527 (3.680832650158761) + | > loss: 0.867391049861908 (0.9295748504551655) + | > current_lr: 5e-06 + | > step_time: 0.3125 (0.3191515652671844) + | > loader_time: 0.0089 (0.009467400693129088) + + + --> TIME: 2024-10-20 07:26:17 -- STEP: 1672/16928 -- GLOBAL_STEP: 18600 + | > loss_text_ce: 0.0378132127225399 (0.03746459672993943) + | > loss_mel_ce: 3.503685235977173 (3.679376389991724) + | > loss: 0.8853746056556702 (0.9292102458588246) + | > current_lr: 5e-06 + | > step_time: 0.2437 (0.3193654995500758) + | > loader_time: 0.0088 (0.009475639704882246) + + + --> TIME: 2024-10-20 07:26:45 -- STEP: 1722/16928 -- GLOBAL_STEP: 18650 + | > loss_text_ce: 0.03649510443210602 (0.037467055254195) + | > loss_mel_ce: 4.146908283233643 (3.679287193821144) + | > loss: 1.0458508729934692 (0.929188561494896) + | > current_lr: 5e-06 + | > step_time: 0.314 (0.31980397753876144) + | > loader_time: 0.0092 (0.009479596916668371) + + + --> TIME: 2024-10-20 07:27:09 -- STEP: 1772/16928 -- GLOBAL_STEP: 18700 + | > loss_text_ce: 0.042395856231451035 (0.037448235255567196) + | > loss_mel_ce: 3.4845895767211914 (3.6780601673955036) + | > loss: 0.8817463517189026 (0.9288770999448295) + | > current_lr: 5e-06 + | > step_time: 0.2725 (0.3188287983091369) + | > loader_time: 0.0103 (0.009473190216124734) + + + --> TIME: 2024-10-20 07:27:37 -- STEP: 1822/16928 -- GLOBAL_STEP: 18750 + | > loss_text_ce: 0.03899775817990303 (0.03745372646767332) + | > loss_mel_ce: 3.558093309402466 (3.6772390030872417) + | > loss: 0.8992727398872375 (0.9286731816542005) + | > current_lr: 5e-06 + | > step_time: 0.2608 (0.3192582186699962) + | > loader_time: 0.0087 (0.0094689804688505) + + + --> TIME: 2024-10-20 07:28:03 -- STEP: 1872/16928 -- GLOBAL_STEP: 18800 + | > loss_text_ce: 0.03957843780517578 (0.03745581841088315) + | > loss_mel_ce: 3.850602865219116 (3.6781265087362027) + | > loss: 0.972545325756073 (0.9288955810997223) + | > current_lr: 5e-06 + | > step_time: 0.2068 (0.3186992237455826) + | > loader_time: 0.0086 (0.00946291962749938) + + + --> TIME: 2024-10-20 07:28:31 -- STEP: 1922/16928 -- GLOBAL_STEP: 18850 + | > loss_text_ce: 0.037315644323825836 (0.037459519519262206) + | > loss_mel_ce: 3.565190076828003 (3.678776119055535) + | > loss: 0.9006264209747314 (0.9290589089200103) + | > current_lr: 5e-06 + | > step_time: 0.4456 (0.3191859113066057) + | > loader_time: 0.0088 (0.009461605213931402) + + + --> TIME: 2024-10-20 07:28:59 -- STEP: 1972/16928 -- GLOBAL_STEP: 18900 + | > loss_text_ce: 0.042596425861120224 (0.03745025009744271) + | > loss_mel_ce: 3.758420944213867 (3.6793897132602713) + | > loss: 0.9502543210983276 (0.9292099901737597) + | > current_lr: 5e-06 + | > step_time: 0.2754 (0.31963287490376613) + | > loader_time: 0.0106 (0.009466333515020223) + + + --> TIME: 2024-10-20 07:29:26 -- STEP: 2022/16928 -- GLOBAL_STEP: 18950 + | > loss_text_ce: 0.03195209056138992 (0.037452574624955354) + | > loss_mel_ce: 3.49176025390625 (3.6789523925554857) + | > loss: 0.880928099155426 (0.9291012410947761) + | > current_lr: 5e-06 + | > step_time: 0.2261 (0.31975513678983714) + | > loader_time: 0.0092 (0.00946656803938567) + + + --> TIME: 2024-10-20 07:29:53 -- STEP: 2072/16928 -- GLOBAL_STEP: 19000 + | > loss_text_ce: 0.03484316170215607 (0.037447761649869925) + | > loss_mel_ce: 3.769008159637451 (3.678837721412246) + | > loss: 0.9509628415107727 (0.9290713700088294) + | > current_lr: 5e-06 + | > step_time: 0.3558 (0.31985109607225237) + | > loader_time: 0.0094 (0.009458464775306381) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_19000.pth + + --> TIME: 2024-10-20 07:30:30 -- STEP: 2122/16928 -- GLOBAL_STEP: 19050 + | > loss_text_ce: 0.04017932713031769 (0.03743597819021319) + | > loss_mel_ce: 3.3342955112457275 (3.677568420051517) + | > loss: 0.8436186909675598 (0.9287510988399963) + | > current_lr: 5e-06 + | > step_time: 0.3989 (0.3195798303809064) + | > loader_time: 0.0097 (0.009465552971792269) + + + --> TIME: 2024-10-20 07:30:58 -- STEP: 2172/16928 -- GLOBAL_STEP: 19100 + | > loss_text_ce: 0.03746473789215088 (0.037438322399771996) + | > loss_mel_ce: 3.589852809906006 (3.6795114943994336) + | > loss: 0.9068293571472168 (0.9292374534830864) + | > current_lr: 5e-06 + | > step_time: 0.2506 (0.3198965069997379) + | > loader_time: 0.009 (0.00946710408282763) + + + --> TIME: 2024-10-20 07:31:24 -- STEP: 2222/16928 -- GLOBAL_STEP: 19150 + | > loss_text_ce: 0.037209682166576385 (0.037432924209751) + | > loss_mel_ce: 3.7231662273406982 (3.678380345890481) + | > loss: 0.940093994140625 (0.9289533168372544) + | > current_lr: 5e-06 + | > step_time: 0.2491 (0.3197172022852999) + | > loader_time: 0.0087 (0.009463654564480653) + + + --> TIME: 2024-10-20 07:31:50 -- STEP: 2272/16928 -- GLOBAL_STEP: 19200 + | > loss_text_ce: 0.034725699573755264 (0.03744508386392822) + | > loss_mel_ce: 3.6324357986450195 (3.6781992416986276) + | > loss: 0.9167903661727905 (0.9289110806626335) + | > current_lr: 5e-06 + | > step_time: 0.2885 (0.3194371578349199) + | > loader_time: 0.0085 (0.009457923366989893) + + + --> TIME: 2024-10-20 07:32:16 -- STEP: 2322/16928 -- GLOBAL_STEP: 19250 + | > loss_text_ce: 0.03842463716864586 (0.03744228538783527) + | > loss_mel_ce: 4.081628799438477 (3.6770015257786923) + | > loss: 1.0300133228302002 (0.9286109519405268) + | > current_lr: 5e-06 + | > step_time: 0.3213 (0.31911986291049216) + | > loader_time: 0.0097 (0.009457048930349694) + + + --> TIME: 2024-10-20 07:32:42 -- STEP: 2372/16928 -- GLOBAL_STEP: 19300 + | > loss_text_ce: 0.03902983292937279 (0.03743907296323349) + | > loss_mel_ce: 3.1799633502960205 (3.677470171029411) + | > loss: 0.8047482967376709 (0.928727310157144) + | > current_lr: 5e-06 + | > step_time: 0.3446 (0.31893123814468693) + | > loader_time: 0.0101 (0.009453619109438406) + + + --> TIME: 2024-10-20 07:33:07 -- STEP: 2422/16928 -- GLOBAL_STEP: 19350 + | > loss_text_ce: 0.03977585583925247 (0.03743713523986461) + | > loss_mel_ce: 4.124311447143555 (3.6775868893850943) + | > loss: 1.0410218238830566 (0.9287560053791124) + | > current_lr: 5e-06 + | > step_time: 0.3958 (0.318595581168877) + | > loader_time: 0.0093 (0.009450448829216182) + + + --> TIME: 2024-10-20 07:33:35 -- STEP: 2472/16928 -- GLOBAL_STEP: 19400 + | > loss_text_ce: 0.03915853798389435 (0.037433549938090484) + | > loss_mel_ce: 3.2896728515625 (3.6776901177796733) + | > loss: 0.8322078585624695 (0.9287809160535005) + | > current_lr: 5e-06 + | > step_time: 0.3184 (0.3188319023951748) + | > loader_time: 0.0102 (0.009446912695289047) + + + --> TIME: 2024-10-20 07:34:01 -- STEP: 2522/16928 -- GLOBAL_STEP: 19450 + | > loss_text_ce: 0.04225969687104225 (0.03742904946559551) + | > loss_mel_ce: 3.576266288757324 (3.677169195911792) + | > loss: 0.904631495475769 (0.9286495605742149) + | > current_lr: 5e-06 + | > step_time: 0.3373 (0.3186184302102378) + | > loader_time: 0.009 (0.009444446624223439) + + + --> TIME: 2024-10-20 07:34:29 -- STEP: 2572/16928 -- GLOBAL_STEP: 19500 + | > loss_text_ce: 0.04284251108765602 (0.03742211406998348) + | > loss_mel_ce: 4.0398173332214355 (3.6769828885376548) + | > loss: 1.020664930343628 (0.9286012499597546) + | > current_lr: 5e-06 + | > step_time: 0.2743 (0.31886374079050234) + | > loader_time: 0.0108 (0.009443126514577207) + + + --> TIME: 2024-10-20 07:34:54 -- STEP: 2622/16928 -- GLOBAL_STEP: 19550 + | > loss_text_ce: 0.036664485931396484 (0.03742276031870613) + | > loss_mel_ce: 3.4304161071777344 (3.677637355915014) + | > loss: 0.8667701482772827 (0.9287650282377511) + | > current_lr: 5e-06 + | > step_time: 0.357 (0.31857521110232073) + | > loader_time: 0.0089 (0.00943836767558921) + + + --> TIME: 2024-10-20 07:35:21 -- STEP: 2672/16928 -- GLOBAL_STEP: 19600 + | > loss_text_ce: 0.03791145980358124 (0.03742995359976693) + | > loss_mel_ce: 3.2964818477630615 (3.6771307949891345) + | > loss: 0.8335983157157898 (0.9286401863628163) + | > current_lr: 5e-06 + | > step_time: 0.1748 (0.31851130188582205) + | > loader_time: 0.0083 (0.009438866894402192) + + + --> TIME: 2024-10-20 07:35:48 -- STEP: 2722/16928 -- GLOBAL_STEP: 19650 + | > loss_text_ce: 0.034927140921354294 (0.03743000748217613) + | > loss_mel_ce: 4.126852035522461 (3.67639374084809) + | > loss: 1.0404448509216309 (0.9284559363043546) + | > current_lr: 5e-06 + | > step_time: 0.3351 (0.31878151506470204) + | > loader_time: 0.0093 (0.009435555060762945) + + + --> TIME: 2024-10-20 07:36:16 -- STEP: 2772/16928 -- GLOBAL_STEP: 19700 + | > loss_text_ce: 0.036738693714141846 (0.03743222559376223) + | > loss_mel_ce: 3.9434380531311035 (3.6771555264274793) + | > loss: 0.9950441718101501 (0.9286469371631894) + | > current_lr: 5e-06 + | > step_time: 0.2879 (0.31887857153646193) + | > loader_time: 0.0106 (0.009432047046452207) + + + --> TIME: 2024-10-20 07:36:42 -- STEP: 2822/16928 -- GLOBAL_STEP: 19750 + | > loss_text_ce: 0.035027824342250824 (0.0374363984953557) + | > loss_mel_ce: 3.3591794967651367 (3.6761628485672193) + | > loss: 0.8485518097877502 (0.9283998109067612) + | > current_lr: 5e-06 + | > step_time: 0.3757 (0.31892936564269103) + | > loader_time: 0.0083 (0.009422357753183227) + + + --> TIME: 2024-10-20 07:37:08 -- STEP: 2872/16928 -- GLOBAL_STEP: 19800 + | > loss_text_ce: 0.041944559663534164 (0.03743449288477457) + | > loss_mel_ce: 3.2485411167144775 (3.6761626157920007) + | > loss: 0.8226214051246643 (0.928399276310023) + | > current_lr: 5e-06 + | > step_time: 0.2432 (0.3186542906468956) + | > loader_time: 0.0088 (0.009410905306717827) + + + --> TIME: 2024-10-20 07:37:34 -- STEP: 2922/16928 -- GLOBAL_STEP: 19850 + | > loss_text_ce: 0.04606565833091736 (0.037426127760220425) + | > loss_mel_ce: 3.397226572036743 (3.6762107003640834) + | > loss: 0.8608230352401733 (0.9284092062634203) + | > current_lr: 5e-06 + | > step_time: 0.2254 (0.3184707668201308) + | > loader_time: 0.0097 (0.009407359634989522) + + + --> TIME: 2024-10-20 07:38:02 -- STEP: 2972/16928 -- GLOBAL_STEP: 19900 + | > loss_text_ce: 0.036495089530944824 (0.037418486744110295) + | > loss_mel_ce: 3.497692346572876 (3.676899112650716) + | > loss: 0.8835468292236328 (0.9285793990337995) + | > current_lr: 5e-06 + | > step_time: 0.2852 (0.31886102509851555) + | > loader_time: 0.0107 (0.009408995809978647) + + + --> TIME: 2024-10-20 07:38:29 -- STEP: 3022/16928 -- GLOBAL_STEP: 19950 + | > loss_text_ce: 0.03777839615941048 (0.03741043802131338) + | > loss_mel_ce: 3.221916437149048 (3.675352256476366) + | > loss: 0.8149237036705017 (0.9281906727763063) + | > current_lr: 5e-06 + | > step_time: 0.4616 (0.3187851158693718) + | > loader_time: 0.0096 (0.009410543050456759) + + + --> TIME: 2024-10-20 07:38:55 -- STEP: 3072/16928 -- GLOBAL_STEP: 20000 + | > loss_text_ce: 0.046832602471113205 (0.03740157961631972) + | > loss_mel_ce: 4.339147090911865 (3.6757073915408305) + | > loss: 1.0964949131011963 (0.9282772419392132) + | > current_lr: 5e-06 + | > step_time: 0.2553 (0.31873058356965545) + | > loader_time: 0.0087 (0.009406077908352047) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_20000.pth + + --> TIME: 2024-10-20 07:39:35 -- STEP: 3122/16928 -- GLOBAL_STEP: 20050 + | > loss_text_ce: 0.03781250864267349 (0.037404951370398204) + | > loss_mel_ce: 3.2737278938293457 (3.67398061277315) + | > loss: 0.827885091304779 (0.9278463901883895) + | > current_lr: 5e-06 + | > step_time: 0.3073 (0.31859664761202944) + | > loader_time: 0.0086 (0.009407317004549009) + + + --> TIME: 2024-10-20 07:40:01 -- STEP: 3172/16928 -- GLOBAL_STEP: 20100 + | > loss_text_ce: 0.036451779305934906 (0.03740655837291655) + | > loss_mel_ce: 3.814736843109131 (3.6734634678420925) + | > loss: 0.9627971649169922 (0.9277175056829129) + | > current_lr: 5e-06 + | > step_time: 0.2082 (0.31848551767019684) + | > loader_time: 0.0089 (0.00940138935892462) + + + --> TIME: 2024-10-20 07:40:29 -- STEP: 3222/16928 -- GLOBAL_STEP: 20150 + | > loss_text_ce: 0.04045262560248375 (0.03740289953232629) + | > loss_mel_ce: 3.594526529312134 (3.6732933854257594) + | > loss: 0.9087448120117188 (0.9276740704538807) + | > current_lr: 5e-06 + | > step_time: 0.4657 (0.31868471232058426) + | > loader_time: 0.0095 (0.009398378399423334) + + + --> TIME: 2024-10-20 07:40:56 -- STEP: 3272/16928 -- GLOBAL_STEP: 20200 + | > loss_text_ce: 0.03638853877782822 (0.037398155615266576) + | > loss_mel_ce: 3.5487117767333984 (3.673694463915813) + | > loss: 0.8962751030921936 (0.9277731541425792) + | > current_lr: 5e-06 + | > step_time: 0.319 (0.3186252466915285) + | > loader_time: 0.0098 (0.009395987524088583) + + + --> TIME: 2024-10-20 07:41:23 -- STEP: 3322/16928 -- GLOBAL_STEP: 20250 + | > loss_text_ce: 0.03545838221907616 (0.03739255592440052) + | > loss_mel_ce: 3.667550802230835 (3.673513174415855) + | > loss: 0.9257522821426392 (0.9277264318647047) + | > current_lr: 5e-06 + | > step_time: 0.3792 (0.3188698061398175) + | > loader_time: 0.009 (0.009396795333687783) + + + --> TIME: 2024-10-20 07:41:50 -- STEP: 3372/16928 -- GLOBAL_STEP: 20300 + | > loss_text_ce: 0.0365070216357708 (0.03738487239381299) + | > loss_mel_ce: 3.600008964538574 (3.6739745187448296) + | > loss: 0.9091290235519409 (0.9278398470981832) + | > current_lr: 5e-06 + | > step_time: 0.3031 (0.31885954783621734) + | > loader_time: 0.0092 (0.00939270617133219) + + + --> TIME: 2024-10-20 07:42:16 -- STEP: 3422/16928 -- GLOBAL_STEP: 20350 + | > loss_text_ce: 0.032143618911504745 (0.0373820928134969) + | > loss_mel_ce: 3.152834892272949 (3.6728805300089995) + | > loss: 0.7962446212768555 (0.9275656550304018) + | > current_lr: 5e-06 + | > step_time: 0.2315 (0.31870590309731617) + | > loader_time: 0.0095 (0.00938609076967581) + + + --> TIME: 2024-10-20 07:42:42 -- STEP: 3472/16928 -- GLOBAL_STEP: 20400 + | > loss_text_ce: 0.03750856593251228 (0.03737613892564678) + | > loss_mel_ce: 3.6112961769104004 (3.6737415253566708) + | > loss: 0.9122011661529541 (0.9277794154592649) + | > current_lr: 5e-06 + | > step_time: 0.3749 (0.31859542456914713) + | > loader_time: 0.0107 (0.009382427700104268) + + + --> TIME: 2024-10-20 07:43:10 -- STEP: 3522/16928 -- GLOBAL_STEP: 20450 + | > loss_text_ce: 0.03782142698764801 (0.03736867812027956) + | > loss_mel_ce: 3.544710159301758 (3.672834354459664) + | > loss: 0.8956329226493835 (0.927550757493166) + | > current_lr: 5e-06 + | > step_time: 0.3803 (0.3188124960759087) + | > loader_time: 0.0103 (0.009381312934056677) + + + --> TIME: 2024-10-20 07:43:36 -- STEP: 3572/16928 -- GLOBAL_STEP: 20500 + | > loss_text_ce: 0.03825979679822922 (0.037366708287967135) + | > loss_mel_ce: 3.82075834274292 (3.6742141834165056) + | > loss: 0.9647545218467712 (0.9278952222555243) + | > current_lr: 5e-06 + | > step_time: 0.2094 (0.31873490655542314) + | > loader_time: 0.009 (0.009378376626487824) + + + --> TIME: 2024-10-20 07:44:03 -- STEP: 3622/16928 -- GLOBAL_STEP: 20550 + | > loss_text_ce: 0.03536941483616829 (0.03735512492778154) + | > loss_mel_ce: 3.5822033882141113 (3.6746086924856765) + | > loss: 0.904393196105957 (0.9279909537254547) + | > current_lr: 5e-06 + | > step_time: 0.1916 (0.31880438637167324) + | > loader_time: 0.0095 (0.009378608190703974) + + + --> TIME: 2024-10-20 07:44:30 -- STEP: 3672/16928 -- GLOBAL_STEP: 20600 + | > loss_text_ce: 0.0391056202352047 (0.0373484666153894) + | > loss_mel_ce: 3.572110414505005 (3.6744208696880634) + | > loss: 0.9028040170669556 (0.927942333398042) + | > current_lr: 5e-06 + | > step_time: 0.2885 (0.3189667275009068) + | > loader_time: 0.0101 (0.009380631197511776) + + + --> TIME: 2024-10-20 07:44:58 -- STEP: 3722/16928 -- GLOBAL_STEP: 20650 + | > loss_text_ce: 0.039359357208013535 (0.03734236996829077) + | > loss_mel_ce: 4.1258392333984375 (3.6747301726620534) + | > loss: 1.0412997007369995 (0.928018134970104) + | > current_lr: 5e-06 + | > step_time: 0.3348 (0.3191963187601536) + | > loader_time: 0.0088 (0.009376790432568729) + + + --> TIME: 2024-10-20 07:45:25 -- STEP: 3772/16928 -- GLOBAL_STEP: 20700 + | > loss_text_ce: 0.039070580154657364 (0.03733252397810492) + | > loss_mel_ce: 3.741755962371826 (3.674560421724208) + | > loss: 0.9452066421508789 (0.9279732357153585) + | > current_lr: 5e-06 + | > step_time: 0.1835 (0.31913440420923367) + | > loader_time: 0.009 (0.009371155392959303) + + + --> TIME: 2024-10-20 07:45:52 -- STEP: 3822/16928 -- GLOBAL_STEP: 20750 + | > loss_text_ce: 0.03936219587922096 (0.03732493108997149) + | > loss_mel_ce: 3.399941921234131 (3.6737724288105027) + | > loss: 0.8598260283470154 (0.9277743392281357) + | > current_lr: 5e-06 + | > step_time: 0.2516 (0.31911503438984484) + | > loader_time: 0.0088 (0.009364063112227984) + + + --> TIME: 2024-10-20 07:46:17 -- STEP: 3872/16928 -- GLOBAL_STEP: 20800 + | > loss_text_ce: 0.03571700677275658 (0.03731791272303397) + | > loss_mel_ce: 3.55364990234375 (3.67372847206829) + | > loss: 0.8973417282104492 (0.9277615954499108) + | > current_lr: 5e-06 + | > step_time: 0.3238 (0.31897904132762195) + | > loader_time: 0.0097 (0.009361498612017686) + + + --> TIME: 2024-10-20 07:46:43 -- STEP: 3922/16928 -- GLOBAL_STEP: 20850 + | > loss_text_ce: 0.03787246719002724 (0.037314751526470964) + | > loss_mel_ce: 3.7439355850219727 (3.6736392199172467) + | > loss: 0.9454520344734192 (0.9277384921581633) + | > current_lr: 5e-06 + | > step_time: 0.3443 (0.3187707443130311) + | > loader_time: 0.0082 (0.009357751297257752) + + + --> TIME: 2024-10-20 07:47:10 -- STEP: 3972/16928 -- GLOBAL_STEP: 20900 + | > loss_text_ce: 0.035452522337436676 (0.03730708201090389) + | > loss_mel_ce: 3.08148455619812 (3.673296467295106) + | > loss: 0.7792342901229858 (0.9276508866300036) + | > current_lr: 5e-06 + | > step_time: 0.3198 (0.3187683282301862) + | > loader_time: 0.0107 (0.009354570180867989) + + + --> TIME: 2024-10-20 07:47:38 -- STEP: 4022/16928 -- GLOBAL_STEP: 20950 + | > loss_text_ce: 0.036304816603660583 (0.037299909054666146) + | > loss_mel_ce: 3.223355770111084 (3.6730716748951324) + | > loss: 0.814915120601654 (0.9275928953137345) + | > current_lr: 5e-06 + | > step_time: 0.5044 (0.3190425163119183) + | > loader_time: 0.01 (0.00935590539863015) + + + --> TIME: 2024-10-20 07:48:04 -- STEP: 4072/16928 -- GLOBAL_STEP: 21000 + | > loss_text_ce: 0.0405355840921402 (0.037301384432658485) + | > loss_mel_ce: 3.837311267852783 (3.6717796763057557) + | > loss: 0.9694617390632629 (0.9272702645459204) + | > current_lr: 5e-06 + | > step_time: 0.1873 (0.31885815802632206) + | > loader_time: 0.0094 (0.009356487769274922) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_21000.pth + + --> TIME: 2024-10-20 07:48:42 -- STEP: 4122/16928 -- GLOBAL_STEP: 21050 + | > loss_text_ce: 0.038578249514102936 (0.03730099933779554) + | > loss_mel_ce: 4.105844497680664 (3.671378393476302) + | > loss: 1.0361056327819824 (0.9271698475664881) + | > current_lr: 5e-06 + | > step_time: 0.3951 (0.3190297982115467) + | > loader_time: 0.0112 (0.009371496820149072) + + + --> TIME: 2024-10-20 07:49:11 -- STEP: 4172/16928 -- GLOBAL_STEP: 21100 + | > loss_text_ce: 0.03714292123913765 (0.037298748159400714) + | > loss_mel_ce: 3.2021679878234863 (3.67080472248284) + | > loss: 0.8098277449607849 (0.927025867102816) + | > current_lr: 5e-06 + | > step_time: 0.1733 (0.319315193254904) + | > loader_time: 0.0102 (0.00937812988961533) + + + --> TIME: 2024-10-20 07:49:37 -- STEP: 4222/16928 -- GLOBAL_STEP: 21150 + | > loss_text_ce: 0.040117353200912476 (0.03728834968077902) + | > loss_mel_ce: 3.519652843475342 (3.6711966331284067) + | > loss: 0.8899425268173218 (0.9271212451721131) + | > current_lr: 5e-06 + | > step_time: 0.1909 (0.3192419343399016) + | > loader_time: 0.0094 (0.009383358869548107) + + + --> TIME: 2024-10-20 07:50:05 -- STEP: 4272/16928 -- GLOBAL_STEP: 21200 + | > loss_text_ce: 0.03472965210676193 (0.037276652460649255) + | > loss_mel_ce: 3.5438735485076904 (3.6711534122737612) + | > loss: 0.8946508169174194 (0.927107515629758) + | > current_lr: 5e-06 + | > step_time: 0.2318 (0.3194943063714528) + | > loader_time: 0.0096 (0.009391212228978632) + + + --> TIME: 2024-10-20 07:50:32 -- STEP: 4322/16928 -- GLOBAL_STEP: 21250 + | > loss_text_ce: 0.04334492236375809 (0.037276205290427834) + | > loss_mel_ce: 3.2877438068389893 (3.671664083461859) + | > loss: 0.8327721953392029 (0.9272350715658613) + | > current_lr: 5e-06 + | > step_time: 0.1302 (0.31938022730914245) + | > loader_time: 0.0094 (0.009398512518113555) + + + --> TIME: 2024-10-20 07:50:58 -- STEP: 4372/16928 -- GLOBAL_STEP: 21300 + | > loss_text_ce: 0.03514856845140457 (0.03727309809987454) + | > loss_mel_ce: 3.4467062950134277 (3.6704465186279625) + | > loss: 0.8704637289047241 (0.9269299035839064) + | > current_lr: 5e-06 + | > step_time: 0.1741 (0.31934741502056807) + | > loader_time: 0.0088 (0.009399865214582559) + + + --> TIME: 2024-10-20 07:51:25 -- STEP: 4422/16928 -- GLOBAL_STEP: 21350 + | > loss_text_ce: 0.0367589145898819 (0.03727258628757427) + | > loss_mel_ce: 3.553983688354492 (3.6708380090656565) + | > loss: 0.8976856470108032 (0.9270276482668145) + | > current_lr: 5e-06 + | > step_time: 0.1917 (0.3193144592857099) + | > loader_time: 0.0093 (0.009403085072403963) + + + --> TIME: 2024-10-20 07:51:50 -- STEP: 4472/16928 -- GLOBAL_STEP: 21400 + | > loss_text_ce: 0.038475096225738525 (0.03727633384479569) + | > loss_mel_ce: 3.3350014686584473 (3.67080081596784) + | > loss: 0.8433691263198853 (0.9270192868940947) + | > current_lr: 5e-06 + | > step_time: 0.2444 (0.31912857086159413) + | > loader_time: 0.0113 (0.009408505850060044) + + + --> TIME: 2024-10-20 07:52:18 -- STEP: 4522/16928 -- GLOBAL_STEP: 21450 + | > loss_text_ce: 0.04090210050344467 (0.03727506134949038) + | > loss_mel_ce: 3.254624605178833 (3.671047070605309) + | > loss: 0.8238816857337952 (0.9270805324246958) + | > current_lr: 5e-06 + | > step_time: 0.3096 (0.31932174547854236) + | > loader_time: 0.0107 (0.009411417504740414) + + + --> TIME: 2024-10-20 07:52:45 -- STEP: 4572/16928 -- GLOBAL_STEP: 21500 + | > loss_text_ce: 0.04022162780165672 (0.0372697760013673) + | > loss_mel_ce: 3.83772873878479 (3.671097623759576) + | > loss: 0.9694876074790955 (0.9270918494056218) + | > current_lr: 5e-06 + | > step_time: 0.1875 (0.319294005658474) + | > loader_time: 0.0094 (0.009415552007036046) + + + --> TIME: 2024-10-20 07:53:13 -- STEP: 4622/16928 -- GLOBAL_STEP: 21550 + | > loss_text_ce: 0.03995572403073311 (0.037262613702547735) + | > loss_mel_ce: 2.9624874591827393 (3.6709747726088837) + | > loss: 0.7506107687950134 (0.9270593460918455) + | > current_lr: 5e-06 + | > step_time: 0.2146 (0.3194228229147176) + | > loader_time: 0.0086 (0.009422563928796427) + + + --> TIME: 2024-10-20 07:53:40 -- STEP: 4672/16928 -- GLOBAL_STEP: 21600 + | > loss_text_ce: 0.038524553179740906 (0.037256150493679) + | > loss_mel_ce: 3.722360849380493 (3.6709043804607164) + | > loss: 0.94022136926651 (0.927040132247422) + | > current_lr: 5e-06 + | > step_time: 0.3757 (0.31946171631347636) + | > loader_time: 0.0093 (0.00942081172172337) + + + --> TIME: 2024-10-20 07:54:06 -- STEP: 4722/16928 -- GLOBAL_STEP: 21650 + | > loss_text_ce: 0.03498861566185951 (0.037254570145120534) + | > loss_mel_ce: 3.8501861095428467 (3.6706696881019982) + | > loss: 0.9712936878204346 (0.9269810640322159) + | > current_lr: 5e-06 + | > step_time: 0.2098 (0.3194242238695465) + | > loader_time: 0.01 (0.009418122145342957) + + + --> TIME: 2024-10-20 07:54:34 -- STEP: 4772/16928 -- GLOBAL_STEP: 21700 + | > loss_text_ce: 0.03386004641652107 (0.03724766387280206) + | > loss_mel_ce: 3.544215679168701 (3.6708877398901794) + | > loss: 0.8945189118385315 (0.9270338503408114) + | > current_lr: 5e-06 + | > step_time: 0.2045 (0.31953051154191797) + | > loader_time: 0.0091 (0.009415986901656584) + + + --> TIME: 2024-10-20 07:55:00 -- STEP: 4822/16928 -- GLOBAL_STEP: 21750 + | > loss_text_ce: 0.03606614097952843 (0.03724601641158282) + | > loss_mel_ce: 4.77860164642334 (3.670877804948007) + | > loss: 1.2036669254302979 (0.9270309547649189) + | > current_lr: 5e-06 + | > step_time: 0.3615 (0.3194103702278563) + | > loader_time: 0.0089 (0.0094146109972194) + + + --> TIME: 2024-10-20 07:55:26 -- STEP: 4872/16928 -- GLOBAL_STEP: 21800 + | > loss_text_ce: 0.039007704704999924 (0.037234071512937505) + | > loss_mel_ce: 4.0405707359313965 (3.67066394088695) + | > loss: 1.0198945999145508 (0.9269745025679789) + | > current_lr: 5e-06 + | > step_time: 0.2509 (0.3193438412814305) + | > loader_time: 0.0092 (0.009412432098623576) + + + --> TIME: 2024-10-20 07:55:52 -- STEP: 4922/16928 -- GLOBAL_STEP: 21850 + | > loss_text_ce: 0.03900596871972084 (0.03722868754185223) + | > loss_mel_ce: 3.5417604446411133 (3.6704667676807854) + | > loss: 0.8951916098594666 (0.9269238632658255) + | > current_lr: 5e-06 + | > step_time: 0.2074 (0.31918466749931423) + | > loader_time: 0.0081 (0.009406985620423485) + + + --> TIME: 2024-10-20 07:56:18 -- STEP: 4972/16928 -- GLOBAL_STEP: 21900 + | > loss_text_ce: 0.03369841352105141 (0.037224646824788904) + | > loss_mel_ce: 3.449784278869629 (3.670117227425042) + | > loss: 0.8708706498146057 (0.9268354679654899) + | > current_lr: 5e-06 + | > step_time: 0.154 (0.3191357455901913) + | > loader_time: 0.0092 (0.009402964830590377) + + + --> TIME: 2024-10-20 07:56:44 -- STEP: 5022/16928 -- GLOBAL_STEP: 21950 + | > loss_text_ce: 0.032713863998651505 (0.03722216286750274) + | > loss_mel_ce: 3.593381404876709 (3.670513674777041) + | > loss: 0.9065238237380981 (0.926933958847558) + | > current_lr: 5e-06 + | > step_time: 0.2324 (0.31911728164675857) + | > loader_time: 0.0108 (0.009398488113482328) + + + --> TIME: 2024-10-20 07:57:10 -- STEP: 5072/16928 -- GLOBAL_STEP: 22000 + | > loss_text_ce: 0.035655345767736435 (0.03721833044930425) + | > loss_mel_ce: 3.4785590171813965 (3.670765142006453) + | > loss: 0.878553569316864 (0.9269958674931189) + | > current_lr: 5e-06 + | > step_time: 0.1887 (0.31898620094786656) + | > loader_time: 0.0078 (0.009397717022369353) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_22000.pth + + --> TIME: 2024-10-20 07:57:50 -- STEP: 5122/16928 -- GLOBAL_STEP: 22050 + | > loss_text_ce: 0.034319549798965454 (0.037215553561195734) + | > loss_mel_ce: 3.788275957107544 (3.6703997640486854) + | > loss: 0.9556488990783691 (0.9269038288098024) + | > current_lr: 5e-06 + | > step_time: 0.2141 (0.31893525455017513) + | > loader_time: 0.0081 (0.009399899836863835) + + + --> TIME: 2024-10-20 07:58:18 -- STEP: 5172/16928 -- GLOBAL_STEP: 22100 + | > loss_text_ce: 0.03385968878865242 (0.03721544015338706) + | > loss_mel_ce: 3.334855318069458 (3.670472817188626) + | > loss: 0.8421787619590759 (0.9269220637242555) + | > current_lr: 5e-06 + | > step_time: 0.1362 (0.31895620266965224) + | > loader_time: 0.0083 (0.009398842447963884) + + + --> TIME: 2024-10-20 07:58:45 -- STEP: 5222/16928 -- GLOBAL_STEP: 22150 + | > loss_text_ce: 0.03362384811043739 (0.03721636959092287) + | > loss_mel_ce: 3.7483632564544678 (3.6697936043434334) + | > loss: 0.9454967975616455 (0.9267524929053025) + | > current_lr: 5e-06 + | > step_time: 0.3795 (0.3190463735243677) + | > loader_time: 0.009 (0.00939502203797802) + + + --> TIME: 2024-10-20 07:59:11 -- STEP: 5272/16928 -- GLOBAL_STEP: 22200 + | > loss_text_ce: 0.03882875293493271 (0.03721397444118266) + | > loss_mel_ce: 3.6682708263397217 (3.670101111593666) + | > loss: 0.9267749190330505 (0.9268287709025224) + | > current_lr: 5e-06 + | > step_time: 0.3735 (0.3190286618534454) + | > loader_time: 0.0108 (0.00939203204081164) + + + --> TIME: 2024-10-20 07:59:39 -- STEP: 5322/16928 -- GLOBAL_STEP: 22250 + | > loss_text_ce: 0.033581361174583435 (0.03720827727399741) + | > loss_mel_ce: 3.699908971786499 (3.6701197690060603) + | > loss: 0.9333725571632385 (0.92683201096462) + | > current_lr: 5e-06 + | > step_time: 0.4406 (0.3191587370797592) + | > loader_time: 0.0093 (0.009391692830673157) + + + --> TIME: 2024-10-20 08:00:07 -- STEP: 5372/16928 -- GLOBAL_STEP: 22300 + | > loss_text_ce: 0.042078953236341476 (0.037202193092815604) + | > loss_mel_ce: 4.080121994018555 (3.6695797646658153) + | > loss: 1.030550241470337 (0.9266954888426718) + | > current_lr: 5e-06 + | > step_time: 0.2014 (0.31929237813267863) + | > loader_time: 0.0077 (0.009386866596858183) + + + --> TIME: 2024-10-20 08:00:34 -- STEP: 5422/16928 -- GLOBAL_STEP: 22350 + | > loss_text_ce: 0.040935248136520386 (0.03720213930534457) + | > loss_mel_ce: 4.582732200622559 (3.669828636494126) + | > loss: 1.1559168100357056 (0.9267576933786555) + | > current_lr: 5e-06 + | > step_time: 0.2892 (0.3193060388779645) + | > loader_time: 0.0089 (0.00938178854470585) + + + --> TIME: 2024-10-20 08:01:02 -- STEP: 5472/16928 -- GLOBAL_STEP: 22400 + | > loss_text_ce: 0.04241817817091942 (0.03719774338625083) + | > loss_mel_ce: 3.8069605827331543 (3.6697206525077597) + | > loss: 0.9623447060585022 (0.9267295983821504) + | > current_lr: 5e-06 + | > step_time: 0.2511 (0.319459081088241) + | > loader_time: 0.0086 (0.009384031009952916) + + + --> TIME: 2024-10-20 08:01:28 -- STEP: 5522/16928 -- GLOBAL_STEP: 22450 + | > loss_text_ce: 0.036537934094667435 (0.03719425648727455) + | > loss_mel_ce: 4.081655025482178 (3.669822298932101) + | > loss: 1.0295482873916626 (0.9267541382978552) + | > current_lr: 5e-06 + | > step_time: 0.2888 (0.31944066400503696) + | > loader_time: 0.0095 (0.009385270124239922) + + + --> TIME: 2024-10-20 08:01:54 -- STEP: 5572/16928 -- GLOBAL_STEP: 22500 + | > loss_text_ce: 0.038977351039648056 (0.037190770945322624) + | > loss_mel_ce: 3.5166125297546387 (3.6696448664569377) + | > loss: 0.8888974785804749 (0.9267089088251501) + | > current_lr: 5e-06 + | > step_time: 0.2121 (0.31931033069421566) + | > loader_time: 0.0087 (0.009390203797038477) + + + --> TIME: 2024-10-20 08:02:22 -- STEP: 5622/16928 -- GLOBAL_STEP: 22550 + | > loss_text_ce: 0.035606250166893005 (0.037187100810203315) + | > loss_mel_ce: 3.646785020828247 (3.6696139268288195) + | > loss: 0.9205977916717529 (0.9267002563911672) + | > current_lr: 5e-06 + | > step_time: 0.4941 (0.31943778799166184) + | > loader_time: 0.0099 (0.009389837031600226) + + + --> TIME: 2024-10-20 08:02:49 -- STEP: 5672/16928 -- GLOBAL_STEP: 22600 + | > loss_text_ce: 0.03753191605210304 (0.03718460367878246) + | > loss_mel_ce: 3.741712808609009 (3.670391065319435) + | > loss: 0.9448111653327942 (0.9268939166903833) + | > current_lr: 5e-06 + | > step_time: 0.1853 (0.3194797890647) + | > loader_time: 0.0082 (0.009386912177412731) + + + --> TIME: 2024-10-20 08:03:14 -- STEP: 5722/16928 -- GLOBAL_STEP: 22650 + | > loss_text_ce: 0.03551580756902695 (0.037185211624261444) + | > loss_mel_ce: 3.6926276683807373 (3.67016770572189) + | > loss: 0.9320358633995056 (0.9268382287908794) + | > current_lr: 5e-06 + | > step_time: 0.3593 (0.31931956553534113) + | > loader_time: 0.0089 (0.00938327805806773) + + + --> TIME: 2024-10-20 08:03:39 -- STEP: 5772/16928 -- GLOBAL_STEP: 22700 + | > loss_text_ce: 0.042215559631586075 (0.037183678579566476) + | > loss_mel_ce: 2.9667434692382812 (3.669922547008292) + | > loss: 0.7522397637367249 (0.9267765558476426) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.3191072587246534) + | > loader_time: 0.0102 (0.009382723811923505) + + + --> TIME: 2024-10-20 08:04:05 -- STEP: 5822/16928 -- GLOBAL_STEP: 22750 + | > loss_text_ce: 0.03969883918762207 (0.03718340655610265) + | > loss_mel_ce: 3.9297711849212646 (3.669799717788801) + | > loss: 0.9923675060272217 (0.9267457805315437) + | > current_lr: 5e-06 + | > step_time: 0.3806 (0.3189821882536216) + | > loader_time: 0.0101 (0.009381967326367039) + + + --> TIME: 2024-10-20 08:04:31 -- STEP: 5872/16928 -- GLOBAL_STEP: 22800 + | > loss_text_ce: 0.037553220987319946 (0.0371797410038501) + | > loss_mel_ce: 3.5067150592803955 (3.6692084566936183) + | > loss: 0.8860670924186707 (0.9265970488457006) + | > current_lr: 5e-06 + | > step_time: 0.3194 (0.31887379304914426) + | > loader_time: 0.0086 (0.009379242275326391) + + + --> TIME: 2024-10-20 08:04:58 -- STEP: 5922/16928 -- GLOBAL_STEP: 22850 + | > loss_text_ce: 0.03634721040725708 (0.03717549966616145) + | > loss_mel_ce: 4.253451347351074 (3.669189275981848) + | > loss: 1.0724496841430664 (0.9265911933116437) + | > current_lr: 5e-06 + | > step_time: 0.2842 (0.3189487932341113) + | > loader_time: 0.0102 (0.00937770629645763) + + + --> TIME: 2024-10-20 08:05:25 -- STEP: 5972/16928 -- GLOBAL_STEP: 22900 + | > loss_text_ce: 0.037254318594932556 (0.03717339215101524) + | > loss_mel_ce: 3.3026721477508545 (3.669195485490325) + | > loss: 0.8349816203117371 (0.9265922188399547) + | > current_lr: 5e-06 + | > step_time: 0.3725 (0.3189549340310704) + | > loader_time: 0.0094 (0.009374323141950134) + + + --> TIME: 2024-10-20 08:05:51 -- STEP: 6022/16928 -- GLOBAL_STEP: 22950 + | > loss_text_ce: 0.03738011047244072 (0.03716955188604908) + | > loss_mel_ce: 3.2888996601104736 (3.6692179871809043) + | > loss: 0.8315699696540833 (0.9265968842151673) + | > current_lr: 5e-06 + | > step_time: 0.445 (0.31893238792463735) + | > loader_time: 0.0083 (0.009371614820416362) + + + --> TIME: 2024-10-20 08:06:19 -- STEP: 6072/16928 -- GLOBAL_STEP: 23000 + | > loss_text_ce: 0.03692910447716713 (0.03716716990862612) + | > loss_mel_ce: 3.753542900085449 (3.6686389568766113) + | > loss: 0.9476180076599121 (0.9264515311468262) + | > current_lr: 5e-06 + | > step_time: 0.3742 (0.3189423225221265) + | > loader_time: 0.0093 (0.009370163966544526) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_23000.pth + + --> TIME: 2024-10-20 08:06:58 -- STEP: 6122/16928 -- GLOBAL_STEP: 23050 + | > loss_text_ce: 0.035057779401540756 (0.03715984831353895) + | > loss_mel_ce: 3.3080198764801025 (3.668373800694884) + | > loss: 0.8357694149017334 (0.9263834117168465) + | > current_lr: 5e-06 + | > step_time: 0.301 (0.31886981375581136) + | > loader_time: 0.0085 (0.009370667020778404) + + + --> TIME: 2024-10-20 08:07:25 -- STEP: 6172/16928 -- GLOBAL_STEP: 23100 + | > loss_text_ce: 0.038192056119441986 (0.03715416695500118) + | > loss_mel_ce: 3.5130181312561035 (3.6678788935064266) + | > loss: 0.8878025412559509 (0.9262582645814913) + | > current_lr: 5e-06 + | > step_time: 0.231 (0.3188584814655724) + | > loader_time: 0.0096 (0.009367151957244272) + + + --> TIME: 2024-10-20 08:07:53 -- STEP: 6222/16928 -- GLOBAL_STEP: 23150 + | > loss_text_ce: 0.03486356511712074 (0.03714982796387355) + | > loss_mel_ce: 3.8295249938964844 (3.667645319410169) + | > loss: 0.9660971164703369 (0.9261987863036071) + | > current_lr: 5e-06 + | > step_time: 0.2099 (0.319060559634494) + | > loader_time: 0.0079 (0.009365317293471818) + + + --> TIME: 2024-10-20 08:08:19 -- STEP: 6272/16928 -- GLOBAL_STEP: 23200 + | > loss_text_ce: 0.03785599395632744 (0.03714660396894488) + | > loss_mel_ce: 3.4324679374694824 (3.6676134538589693) + | > loss: 0.8675810098648071 (0.9261900138927207) + | > current_lr: 5e-06 + | > step_time: 0.3559 (0.3189271601791286) + | > loader_time: 0.0133 (0.009362110982135843) + + + --> TIME: 2024-10-20 08:08:45 -- STEP: 6322/16928 -- GLOBAL_STEP: 23250 + | > loss_text_ce: 0.035924043506383896 (0.037138844832273706) + | > loss_mel_ce: 3.9573068618774414 (3.6677430632445525) + | > loss: 0.9983077049255371 (0.9262204764670717) + | > current_lr: 5e-06 + | > step_time: 0.1899 (0.31883522454261504) + | > loader_time: 0.0089 (0.009360693142935278) + + + --> TIME: 2024-10-20 08:09:11 -- STEP: 6372/16928 -- GLOBAL_STEP: 23300 + | > loss_text_ce: 0.03323210030794144 (0.03713397546370257) + | > loss_mel_ce: 4.0897040367126465 (3.667571277717188) + | > loss: 1.0307340621948242 (0.92617631273792) + | > current_lr: 5e-06 + | > step_time: 0.1759 (0.3188163443922775) + | > loader_time: 0.0092 (0.00935835663956512) + + + --> TIME: 2024-10-20 08:09:38 -- STEP: 6422/16928 -- GLOBAL_STEP: 23350 + | > loss_text_ce: 0.035826850682497025 (0.03713130136433801) + | > loss_mel_ce: 2.992234706878662 (3.6668793627570553) + | > loss: 0.7570154070854187 (0.9260026654956574) + | > current_lr: 5e-06 + | > step_time: 0.2988 (0.3188301257276345) + | > loader_time: 0.0079 (0.009355139598709813) + + + --> TIME: 2024-10-20 08:10:05 -- STEP: 6472/16928 -- GLOBAL_STEP: 23400 + | > loss_text_ce: 0.03622475638985634 (0.037127397915323185) + | > loss_mel_ce: 3.9803178310394287 (3.6668991050113116) + | > loss: 1.0041356086730957 (0.9260066252187983) + | > current_lr: 5e-06 + | > step_time: 0.3576 (0.3188526267174298) + | > loader_time: 0.0092 (0.009353352086653074) + + + --> TIME: 2024-10-20 08:10:32 -- STEP: 6522/16928 -- GLOBAL_STEP: 23450 + | > loss_text_ce: 0.039978645741939545 (0.037119151598281906) + | > loss_mel_ce: 3.4596402645111084 (3.6666552661131293) + | > loss: 0.8749047517776489 (0.925943603913355) + | > current_lr: 5e-06 + | > step_time: 0.2508 (0.3189119666092825) + | > loader_time: 0.0084 (0.009349769150980469) + + + --> TIME: 2024-10-20 08:10:57 -- STEP: 6572/16928 -- GLOBAL_STEP: 23500 + | > loss_text_ce: 0.0341903455555439 (0.03711336721681471) + | > loss_mel_ce: 3.1141157150268555 (3.6659639025343966) + | > loss: 0.7870765328407288 (0.9257693169270068) + | > current_lr: 5e-06 + | > step_time: 0.2321 (0.3187106125832187) + | > loader_time: 0.0088 (0.009346479224636267) + + + --> TIME: 2024-10-20 08:11:24 -- STEP: 6622/16928 -- GLOBAL_STEP: 23550 + | > loss_text_ce: 0.03417832404375076 (0.03711120962586946) + | > loss_mel_ce: 3.5107572078704834 (3.6660193604884355) + | > loss: 0.8862338662147522 (0.925782642023184) + | > current_lr: 5e-06 + | > step_time: 0.2263 (0.31878177740177144) + | > loader_time: 0.008 (0.009345561670990567) + + + --> TIME: 2024-10-20 08:11:52 -- STEP: 6672/16928 -- GLOBAL_STEP: 23600 + | > loss_text_ce: 0.03562642261385918 (0.03711059770845023) + | > loss_mel_ce: 3.7764627933502197 (3.666604728328524) + | > loss: 0.953022301197052 (0.9259288309843279) + | > current_lr: 5e-06 + | > step_time: 0.3479 (0.3188748768836763) + | > loader_time: 0.0094 (0.009343001982576819) + + + --> TIME: 2024-10-20 08:12:19 -- STEP: 6722/16928 -- GLOBAL_STEP: 23650 + | > loss_text_ce: 0.043991174548864365 (0.03710462529006089) + | > loss_mel_ce: 3.939667224884033 (3.666471206017528) + | > loss: 0.9959145784378052 (0.9258939572935552) + | > current_lr: 5e-06 + | > step_time: 0.2319 (0.3189621676082097) + | > loader_time: 0.0092 (0.009340153035575565) + + + --> TIME: 2024-10-20 08:12:46 -- STEP: 6772/16928 -- GLOBAL_STEP: 23700 + | > loss_text_ce: 0.04028848186135292 (0.03710058473333361) + | > loss_mel_ce: 3.9032859802246094 (3.666135569049702) + | > loss: 0.9858936071395874 (0.9258090379019145) + | > current_lr: 5e-06 + | > step_time: 0.2068 (0.318929590491663) + | > loader_time: 0.008 (0.009338069194460192) + + + --> TIME: 2024-10-20 08:13:12 -- STEP: 6822/16928 -- GLOBAL_STEP: 23750 + | > loss_text_ce: 0.03743062540888786 (0.03710081691085671) + | > loss_mel_ce: 3.595271348953247 (3.6662341918598624) + | > loss: 0.9081754684448242 (0.925833751626883) + | > current_lr: 5e-06 + | > step_time: 0.4278 (0.31889091041982115) + | > loader_time: 0.0097 (0.009337261323528986) + + + --> TIME: 2024-10-20 08:13:39 -- STEP: 6872/16928 -- GLOBAL_STEP: 23800 + | > loss_text_ce: 0.03649308532476425 (0.037100543088319976) + | > loss_mel_ce: 3.6689014434814453 (3.6660938810760957) + | > loss: 0.9263486266136169 (0.9257986054813211) + | > current_lr: 5e-06 + | > step_time: 0.1744 (0.318897039600246) + | > loader_time: 0.0078 (0.009337442787201458) + + + --> TIME: 2024-10-20 08:14:05 -- STEP: 6922/16928 -- GLOBAL_STEP: 23850 + | > loss_text_ce: 0.03679782897233963 (0.03709499497769588) + | > loss_mel_ce: 3.873751640319824 (3.6660290513552534) + | > loss: 0.9776373505592346 (0.9257810110044907) + | > current_lr: 5e-06 + | > step_time: 0.3064 (0.3188302382951666) + | > loader_time: 0.0088 (0.009336351693899255) + + + --> TIME: 2024-10-20 08:14:32 -- STEP: 6972/16928 -- GLOBAL_STEP: 23900 + | > loss_text_ce: 0.03709283098578453 (0.03709173353540674) + | > loss_mel_ce: 3.698897361755371 (3.6656157137399696) + | > loss: 0.9339975714683533 (0.9256768612446488) + | > current_lr: 5e-06 + | > step_time: 0.4073 (0.3189320290013424) + | > loader_time: 0.0095 (0.00933436484290342) + + + --> TIME: 2024-10-20 08:14:58 -- STEP: 7022/16928 -- GLOBAL_STEP: 23950 + | > loss_text_ce: 0.03348274528980255 (0.037085088373622815) + | > loss_mel_ce: 3.3686604499816895 (3.6650059068348972) + | > loss: 0.8505358099937439 (0.9255227481952186) + | > current_lr: 5e-06 + | > step_time: 0.277 (0.31884975760657397) + | > loader_time: 0.0093 (0.009331851017777275) + + + --> TIME: 2024-10-20 08:15:26 -- STEP: 7072/16928 -- GLOBAL_STEP: 24000 + | > loss_text_ce: 0.037345319986343384 (0.03707994510603002) + | > loss_mel_ce: 3.5995073318481445 (3.665258662328461) + | > loss: 0.9092131853103638 (0.925584651201087) + | > current_lr: 5e-06 + | > step_time: 0.1831 (0.31892250211934725) + | > loader_time: 0.008 (0.009332473158027215) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_24000.pth + + --> TIME: 2024-10-20 08:16:03 -- STEP: 7122/16928 -- GLOBAL_STEP: 24050 + | > loss_text_ce: 0.03913309797644615 (0.03707477688654437) + | > loss_mel_ce: 4.11806583404541 (3.665570013813007) + | > loss: 1.039299726486206 (0.9256611970394257) + | > current_lr: 5e-06 + | > step_time: 0.2537 (0.3188872525143646) + | > loader_time: 0.0089 (0.009334850097030608) + + + --> TIME: 2024-10-20 08:16:30 -- STEP: 7172/16928 -- GLOBAL_STEP: 24100 + | > loss_text_ce: 0.036733753979206085 (0.03707278874555776) + | > loss_mel_ce: 4.011746883392334 (3.6647402689035418) + | > loss: 1.0121201276779175 (0.9254532637904627) + | > current_lr: 5e-06 + | > step_time: 0.3308 (0.3188672503212348) + | > loader_time: 0.009 (0.009331038326639968) + + + --> TIME: 2024-10-20 08:16:56 -- STEP: 7222/16928 -- GLOBAL_STEP: 24150 + | > loss_text_ce: 0.03456959128379822 (0.03706998592183299) + | > loss_mel_ce: 3.8963279724121094 (3.6642382625594623) + | > loss: 0.9827243685722351 (0.9253270615173886) + | > current_lr: 5e-06 + | > step_time: 0.2103 (0.318861735392) + | > loader_time: 0.008 (0.009328676009567678) + + + --> TIME: 2024-10-20 08:17:24 -- STEP: 7272/16928 -- GLOBAL_STEP: 24200 + | > loss_text_ce: 0.03729865327477455 (0.0370662300373615) + | > loss_mel_ce: 3.9220592975616455 (3.664066599245512) + | > loss: 0.989839494228363 (0.9252832067025902) + | > current_lr: 5e-06 + | > step_time: 0.3544 (0.3189363024546895) + | > loader_time: 0.0087 (0.009326830817790148) + + + --> TIME: 2024-10-20 08:17:50 -- STEP: 7322/16928 -- GLOBAL_STEP: 24250 + | > loss_text_ce: 0.03686434030532837 (0.03706599281766548) + | > loss_mel_ce: 3.550804376602173 (3.6640085483697944) + | > loss: 0.8969171643257141 (0.9252686346449901) + | > current_lr: 5e-06 + | > step_time: 0.4472 (0.3188364759425395) + | > loader_time: 0.009 (0.009321208568063434) + + + --> TIME: 2024-10-20 08:18:16 -- STEP: 7372/16928 -- GLOBAL_STEP: 24300 + | > loss_text_ce: 0.03498837724328041 (0.03705947376318736) + | > loss_mel_ce: 3.490473985671997 (3.6634332677688017) + | > loss: 0.8813655972480774 (0.9251231847172255) + | > current_lr: 5e-06 + | > step_time: 0.3451 (0.3188151983676629) + | > loader_time: 0.0085 (0.009317875587610302) + + + --> TIME: 2024-10-20 08:18:42 -- STEP: 7422/16928 -- GLOBAL_STEP: 24350 + | > loss_text_ce: 0.03477615863084793 (0.03705353611526613) + | > loss_mel_ce: 3.5035746097564697 (3.663492348662479) + | > loss: 0.8845877051353455 (0.9251364705633907) + | > current_lr: 5e-06 + | > step_time: 0.3587 (0.3187875753462847) + | > loader_time: 0.0089 (0.009316577290756203) + + + --> TIME: 2024-10-20 08:19:09 -- STEP: 7472/16928 -- GLOBAL_STEP: 24400 + | > loss_text_ce: 0.03738619014620781 (0.03705119488838625) + | > loss_mel_ce: 3.319467067718506 (3.6628112267758657) + | > loss: 0.8392133116722107 (0.9249656047648104) + | > current_lr: 5e-06 + | > step_time: 0.2492 (0.3187696693477082) + | > loader_time: 0.0087 (0.00931487042592472) + + + --> TIME: 2024-10-20 08:19:37 -- STEP: 7522/16928 -- GLOBAL_STEP: 24450 + | > loss_text_ce: 0.03732912242412567 (0.037049462102557815) + | > loss_mel_ce: 3.5729925632476807 (3.662974327018178) + | > loss: 0.9025804400444031 (0.9250059466322698) + | > current_lr: 5e-06 + | > step_time: 0.2572 (0.3188552104929161) + | > loader_time: 0.0083 (0.009315289198417193) + + + --> TIME: 2024-10-20 08:20:04 -- STEP: 7572/16928 -- GLOBAL_STEP: 24500 + | > loss_text_ce: 0.03706016764044762 (0.037045816676883246) + | > loss_mel_ce: 3.4522957801818848 (3.663050347324156) + | > loss: 0.8723390102386475 (0.9250240403621587) + | > current_lr: 5e-06 + | > step_time: 0.2103 (0.31886829907341246) + | > loader_time: 0.0095 (0.009313217811463529) + + + --> TIME: 2024-10-20 08:20:30 -- STEP: 7622/16928 -- GLOBAL_STEP: 24550 + | > loss_text_ce: 0.035581160336732864 (0.03704122290586106) + | > loss_mel_ce: 3.426600933074951 (3.6635030229333756) + | > loss: 0.8655455112457275 (0.9251360608442223) + | > current_lr: 5e-06 + | > step_time: 0.4016 (0.3188950648554292) + | > loader_time: 0.0089 (0.00931159976397681) + + + --> TIME: 2024-10-20 08:20:57 -- STEP: 7672/16928 -- GLOBAL_STEP: 24600 + | > loss_text_ce: 0.03592958673834801 (0.03703646571885297) + | > loss_mel_ce: 3.6067185401916504 (3.6631163729741254) + | > loss: 0.910662055015564 (0.9250382090587562) + | > current_lr: 5e-06 + | > step_time: 0.1833 (0.31889223449545956) + | > loader_time: 0.0084 (0.00930841461336771) + + + --> TIME: 2024-10-20 08:21:24 -- STEP: 7722/16928 -- GLOBAL_STEP: 24650 + | > loss_text_ce: 0.0372905358672142 (0.037032493821672785) + | > loss_mel_ce: 3.668017864227295 (3.662756782249909) + | > loss: 0.926327109336853 (0.9249473184123308) + | > current_lr: 5e-06 + | > step_time: 0.2484 (0.3188940945822662) + | > loader_time: 0.0081 (0.009305672953086173) + + + --> TIME: 2024-10-20 08:21:51 -- STEP: 7772/16928 -- GLOBAL_STEP: 24700 + | > loss_text_ce: 0.03735261410474777 (0.037031942522666325) + | > loss_mel_ce: 3.770829916000366 (3.662792030463069) + | > loss: 0.9520456194877625 (0.9249559926253524) + | > current_lr: 5e-06 + | > step_time: 0.3203 (0.31895413158851643) + | > loader_time: 0.0091 (0.009303847570365288) + + + --> TIME: 2024-10-20 08:22:18 -- STEP: 7822/16928 -- GLOBAL_STEP: 24750 + | > loss_text_ce: 0.037478797137737274 (0.037027624933650784) + | > loss_mel_ce: 3.6586809158325195 (3.662263566104691) + | > loss: 0.924039900302887 (0.9248227971239001) + | > current_lr: 5e-06 + | > step_time: 0.3975 (0.3190281373465103) + | > loader_time: 0.0093 (0.00930249992766605) + + + --> TIME: 2024-10-20 08:22:45 -- STEP: 7872/16928 -- GLOBAL_STEP: 24800 + | > loss_text_ce: 0.03395809233188629 (0.037021691953040455) + | > loss_mel_ce: 3.4862658977508545 (3.6621666107235886) + | > loss: 0.8800560235977173 (0.9247970750585685) + | > current_lr: 5e-06 + | > step_time: 0.3544 (0.31898605693283144) + | > loader_time: 0.01 (0.009302398023324252) + + + --> TIME: 2024-10-20 08:23:14 -- STEP: 7922/16928 -- GLOBAL_STEP: 24850 + | > loss_text_ce: 0.037521522492170334 (0.037016846394277984) + | > loss_mel_ce: 3.3915164470672607 (3.6619311292133556) + | > loss: 0.8572595119476318 (0.9247369932815362) + | > current_lr: 5e-06 + | > step_time: 0.5811 (0.31917945169734424) + | > loader_time: 0.0119 (0.00930351300300955) + + + --> TIME: 2024-10-20 08:23:41 -- STEP: 7972/16928 -- GLOBAL_STEP: 24900 + | > loss_text_ce: 0.034203555434942245 (0.0370119547871412) + | > loss_mel_ce: 3.071530818939209 (3.6615527257569997) + | > loss: 0.7764335870742798 (0.9246411695291336) + | > current_lr: 5e-06 + | > step_time: 0.2028 (0.3192268883649634) + | > loader_time: 0.0089 (0.009303597338045264) + + + --> TIME: 2024-10-20 08:24:09 -- STEP: 8022/16928 -- GLOBAL_STEP: 24950 + | > loss_text_ce: 0.03605655953288078 (0.03700676204358545) + | > loss_mel_ce: 3.4121384620666504 (3.6614993931319284) + | > loss: 0.8620487451553345 (0.924626538199815) + | > current_lr: 5e-06 + | > step_time: 0.2122 (0.3192821094978485) + | > loader_time: 0.008 (0.00930446358102716) + + + --> TIME: 2024-10-20 08:24:36 -- STEP: 8072/16928 -- GLOBAL_STEP: 25000 + | > loss_text_ce: 0.03455669432878494 (0.03700264476630135) + | > loss_mel_ce: 3.627183437347412 (3.6611717259742105) + | > loss: 0.9154350161552429 (0.9245435920930714) + | > current_lr: 5e-06 + | > step_time: 0.351 (0.31929864965534327) + | > loader_time: 0.0101 (0.00930317060798551) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_25000.pth + + --> TIME: 2024-10-20 08:25:15 -- STEP: 8122/16928 -- GLOBAL_STEP: 25050 + | > loss_text_ce: 0.041926898062229156 (0.03699659512898192) + | > loss_mel_ce: 3.619178295135498 (3.6607487005014425) + | > loss: 0.915276288986206 (0.924436323331865) + | > current_lr: 5e-06 + | > step_time: 0.3765 (0.31940192378996884) + | > loader_time: 0.0087 (0.009304939629085747) + + + --> TIME: 2024-10-20 08:25:42 -- STEP: 8172/16928 -- GLOBAL_STEP: 25100 + | > loss_text_ce: 0.03436298668384552 (0.03699359390246452) + | > loss_mel_ce: 3.6043057441711426 (3.6605444428270726) + | > loss: 0.9096671938896179 (0.9243845086046953) + | > current_lr: 5e-06 + | > step_time: 0.1501 (0.31945577872132086) + | > loader_time: 0.0094 (0.009301649170416653) + + + --> TIME: 2024-10-20 08:26:10 -- STEP: 8222/16928 -- GLOBAL_STEP: 25150 + | > loss_text_ce: 0.039831604808568954 (0.0369877011975766) + | > loss_mel_ce: 3.5149970054626465 (3.65996568207612) + | > loss: 0.888707160949707 (0.9242383452466272) + | > current_lr: 5e-06 + | > step_time: 0.3014 (0.3195577189163177) + | > loader_time: 0.0078 (0.009299202720430493) + + + --> TIME: 2024-10-20 08:26:37 -- STEP: 8272/16928 -- GLOBAL_STEP: 25200 + | > loss_text_ce: 0.03545698896050453 (0.03698317258262113) + | > loss_mel_ce: 3.1629037857055664 (3.659459165551446) + | > loss: 0.7995901703834534 (0.9241105839538045) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.31959563534079966) + | > loader_time: 0.0079 (0.009297285913498736) + + + --> TIME: 2024-10-20 08:27:05 -- STEP: 8322/16928 -- GLOBAL_STEP: 25250 + | > loss_text_ce: 0.03495512902736664 (0.03697774011701742) + | > loss_mel_ce: 3.5495779514312744 (3.6590447922619282) + | > loss: 0.8961332440376282 (0.9240056325168287) + | > current_lr: 5e-06 + | > step_time: 0.3283 (0.3196895353727061) + | > loader_time: 0.0087 (0.009296039783218305) + + + --> TIME: 2024-10-20 08:27:32 -- STEP: 8372/16928 -- GLOBAL_STEP: 25300 + | > loss_text_ce: 0.0367501936852932 (0.036973426898970936) + | > loss_mel_ce: 3.5293691158294678 (3.65914173138751) + | > loss: 0.8915297985076904 (0.9240287889491071) + | > current_lr: 5e-06 + | > step_time: 0.2382 (0.3197129655556423) + | > loader_time: 0.0084 (0.009294070812485943) + + + --> TIME: 2024-10-20 08:27:59 -- STEP: 8422/16928 -- GLOBAL_STEP: 25350 + | > loss_text_ce: 0.035138681530952454 (0.03697103977480284) + | > loss_mel_ce: 3.3109254837036133 (3.6589975873129124) + | > loss: 0.8365160226821899 (0.9239921561119194) + | > current_lr: 5e-06 + | > step_time: 0.2501 (0.3196831439119485) + | > loader_time: 0.0079 (0.009290316104322707) + + + --> TIME: 2024-10-20 08:28:24 -- STEP: 8472/16928 -- GLOBAL_STEP: 25400 + | > loss_text_ce: 0.03679976612329483 (0.03696867217493591) + | > loss_mel_ce: 3.6351025104522705 (3.6586977378786223) + | > loss: 0.9179755449295044 (0.9239166018406212) + | > current_lr: 5e-06 + | > step_time: 0.2032 (0.3195790764123796) + | > loader_time: 0.008 (0.009284963913972448) + + + --> TIME: 2024-10-20 08:28:52 -- STEP: 8522/16928 -- GLOBAL_STEP: 25450 + | > loss_text_ce: 0.035684145987033844 (0.03696475701257551) + | > loss_mel_ce: 3.8600945472717285 (3.658191245252944) + | > loss: 0.9739446640014648 (0.9237889998942802) + | > current_lr: 5e-06 + | > step_time: 0.4399 (0.31973526050327716) + | > loader_time: 0.0092 (0.009281362314454745) + + + --> TIME: 2024-10-20 08:29:20 -- STEP: 8572/16928 -- GLOBAL_STEP: 25500 + | > loss_text_ce: 0.03820526599884033 (0.03696141314722798) + | > loss_mel_ce: 3.3690085411071777 (3.6583038225554434) + | > loss: 0.8518034219741821 (0.9238163082401049) + | > current_lr: 5e-06 + | > step_time: 0.246 (0.3198012308266695) + | > loader_time: 0.0095 (0.009278653160857056) + + + --> TIME: 2024-10-20 08:29:46 -- STEP: 8622/16928 -- GLOBAL_STEP: 25550 + | > loss_text_ce: 0.03594315052032471 (0.03695820046623791) + | > loss_mel_ce: 3.6487770080566406 (3.6582142163307414) + | > loss: 0.921180009841919 (0.9237931034759091) + | > current_lr: 5e-06 + | > step_time: 0.3099 (0.31978397973865197) + | > loader_time: 0.0085 (0.009276432217868042) + + + --> TIME: 2024-10-20 08:30:13 -- STEP: 8672/16928 -- GLOBAL_STEP: 25600 + | > loss_text_ce: 0.036440249532461166 (0.036953589458205975) + | > loss_mel_ce: 3.8431942462921143 (3.6584615176509225) + | > loss: 0.969908595085144 (0.9238537760499549) + | > current_lr: 5e-06 + | > step_time: 0.2223 (0.31977201266922206) + | > loader_time: 0.0092 (0.009273975372754312) + + + --> TIME: 2024-10-20 08:30:40 -- STEP: 8722/16928 -- GLOBAL_STEP: 25650 + | > loss_text_ce: 0.03679436072707176 (0.03694834842884145) + | > loss_mel_ce: 3.579904794692993 (3.6581558135091043) + | > loss: 0.9041748046875 (0.9237760397798013) + | > current_lr: 5e-06 + | > step_time: 0.4405 (0.31977443242942327) + | > loader_time: 0.0084 (0.009271073904477) + + + --> TIME: 2024-10-20 08:31:05 -- STEP: 8772/16928 -- GLOBAL_STEP: 25700 + | > loss_text_ce: 0.037274304777383804 (0.03694580922755174) + | > loss_mel_ce: 3.398703098297119 (3.6578597348137523) + | > loss: 0.8589943647384644 (0.9237013853172489) + | > current_lr: 5e-06 + | > step_time: 0.1515 (0.3196385505261404) + | > loader_time: 0.0081 (0.009266962947445351) + + + --> TIME: 2024-10-20 08:31:30 -- STEP: 8822/16928 -- GLOBAL_STEP: 25750 + | > loss_text_ce: 0.03480282798409462 (0.03694375793147036) + | > loss_mel_ce: 4.350925445556641 (3.6576390735580384) + | > loss: 1.096432089805603 (0.923645707160214) + | > current_lr: 5e-06 + | > step_time: 0.431 (0.31951264523238304) + | > loader_time: 0.0085 (0.009263404451835915) + + + --> TIME: 2024-10-20 08:31:56 -- STEP: 8872/16928 -- GLOBAL_STEP: 25800 + | > loss_text_ce: 0.03701397031545639 (0.036941228843419355) + | > loss_mel_ce: 3.9094905853271484 (3.6573779970219373) + | > loss: 0.986626148223877 (0.923579805741394) + | > current_lr: 5e-06 + | > step_time: 0.2016 (0.3194980393991051) + | > loader_time: 0.0091 (0.009261213437837965) + + + --> TIME: 2024-10-20 08:32:24 -- STEP: 8922/16928 -- GLOBAL_STEP: 25850 + | > loss_text_ce: 0.03704466670751572 (0.03693775492065826) + | > loss_mel_ce: 3.1974740028381348 (3.6578258462776336) + | > loss: 0.8086296916007996 (0.9236908996153295) + | > current_lr: 5e-06 + | > step_time: 0.3067 (0.3195641126314158) + | > loader_time: 0.0089 (0.009259748047335028) + + + --> TIME: 2024-10-20 08:32:51 -- STEP: 8972/16928 -- GLOBAL_STEP: 25900 + | > loss_text_ce: 0.03526463732123375 (0.03693501822645467) + | > loss_mel_ce: 3.773900270462036 (3.6575701202710613) + | > loss: 0.9522912502288818 (0.923626283915299) + | > current_lr: 5e-06 + | > step_time: 0.4193 (0.31958153629962016) + | > loader_time: 0.0102 (0.00925729958223433) + + + --> TIME: 2024-10-20 08:33:19 -- STEP: 9022/16928 -- GLOBAL_STEP: 25950 + | > loss_text_ce: 0.03976460546255112 (0.03693188279264337) + | > loss_mel_ce: 4.031726837158203 (3.6569908271209015) + | > loss: 1.0178728103637695 (0.9234806767616746) + | > current_lr: 5e-06 + | > step_time: 0.3348 (0.3196639503562725) + | > loader_time: 0.0086 (0.009254312837660301) + + + --> TIME: 2024-10-20 08:33:43 -- STEP: 9072/16928 -- GLOBAL_STEP: 26000 + | > loss_text_ce: 0.034340474754571915 (0.03692704131062991) + | > loss_mel_ce: 3.908057928085327 (3.6569281260908384) + | > loss: 0.9855995774269104 (0.9234637911357577) + | > current_lr: 5e-06 + | > step_time: 0.2882 (0.319511940108195) + | > loader_time: 0.0084 (0.009250493250406161) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_26000.pth + + --> TIME: 2024-10-20 08:34:21 -- STEP: 9122/16928 -- GLOBAL_STEP: 26050 + | > loss_text_ce: 0.03621649369597435 (0.03692640854967491) + | > loss_mel_ce: 3.4425244331359863 (3.6569067474468526) + | > loss: 0.8696852326393127 (0.9234582882940547) + | > current_lr: 5e-06 + | > step_time: 0.4444 (0.31949121649917134) + | > loader_time: 0.0093 (0.009250275321865075) + + + --> TIME: 2024-10-20 08:34:47 -- STEP: 9172/16928 -- GLOBAL_STEP: 26100 + | > loss_text_ce: 0.032632969319820404 (0.03691966848661574) + | > loss_mel_ce: 3.819491147994995 (3.656772572584564) + | > loss: 0.9630310535430908 (0.9234230595608751) + | > current_lr: 5e-06 + | > step_time: 0.3585 (0.3194524225479537) + | > loader_time: 0.0088 (0.00924816773128049) + + + --> TIME: 2024-10-20 08:35:11 -- STEP: 9222/16928 -- GLOBAL_STEP: 26150 + | > loss_text_ce: 0.036446090787649155 (0.036918488393954556) + | > loss_mel_ce: 3.4398157596588135 (3.656400920977124) + | > loss: 0.8690654635429382 (0.923329851608477) + | > current_lr: 5e-06 + | > step_time: 0.3984 (0.31926250000161294) + | > loader_time: 0.0102 (0.009246042146519315) + + + --> TIME: 2024-10-20 08:35:37 -- STEP: 9272/16928 -- GLOBAL_STEP: 26200 + | > loss_text_ce: 0.03872902691364288 (0.03691530927840547) + | > loss_mel_ce: 4.058322906494141 (3.6561207627303856) + | > loss: 1.0242630243301392 (0.9232590172738737) + | > current_lr: 5e-06 + | > step_time: 0.2717 (0.31920638518460936) + | > loader_time: 0.0088 (0.009243116644150813) + + + --> TIME: 2024-10-20 08:36:05 -- STEP: 9322/16928 -- GLOBAL_STEP: 26250 + | > loss_text_ce: 0.03489677235484123 (0.03691403429001754) + | > loss_mel_ce: 4.059609889984131 (3.656164408139317) + | > loss: 1.023626685142517 (0.923269609874524) + | > current_lr: 5e-06 + | > step_time: 0.414 (0.31928284760069786) + | > loader_time: 0.0104 (0.009242473127401502) + + + --> TIME: 2024-10-20 08:36:31 -- STEP: 9372/16928 -- GLOBAL_STEP: 26300 + | > loss_text_ce: 0.03319065272808075 (0.036912111724069926) + | > loss_mel_ce: 4.147007465362549 (3.6561667515434775) + | > loss: 1.0450495481491089 (0.9232697150822231) + | > current_lr: 5e-06 + | > step_time: 0.3747 (0.31924974872574424) + | > loader_time: 0.0091 (0.009240889747683095) + + + --> TIME: 2024-10-20 08:37:00 -- STEP: 9422/16928 -- GLOBAL_STEP: 26350 + | > loss_text_ce: 0.03602851182222366 (0.03690682910619988) + | > loss_mel_ce: 3.617917776107788 (3.656469992254221) + | > loss: 0.9134865999221802 (0.923344204580675) + | > current_lr: 5e-06 + | > step_time: 0.4673 (0.3193836293132456) + | > loader_time: 0.0091 (0.009238934193009682) + + + --> TIME: 2024-10-20 08:37:28 -- STEP: 9472/16928 -- GLOBAL_STEP: 26400 + | > loss_text_ce: 0.03748812898993492 (0.03690422911628009) + | > loss_mel_ce: 3.811422348022461 (3.656780985511235) + | > loss: 0.9622276425361633 (0.9234213028795313) + | > current_lr: 5e-06 + | > step_time: 0.2301 (0.31947904800039645) + | > loader_time: 0.0086 (0.009237904932249207) + + + --> TIME: 2024-10-20 08:37:56 -- STEP: 9522/16928 -- GLOBAL_STEP: 26450 + | > loss_text_ce: 0.035489633679389954 (0.036899834791090144) + | > loss_mel_ce: 4.043118953704834 (3.656679435853972) + | > loss: 1.0196521282196045 (0.9233948168972915) + | > current_lr: 5e-06 + | > step_time: 0.4287 (0.31952726360229666) + | > loader_time: 0.0095 (0.009239458707269782) + + + --> TIME: 2024-10-20 08:38:23 -- STEP: 9572/16928 -- GLOBAL_STEP: 26500 + | > loss_text_ce: 0.03803984075784683 (0.03689685952878887) + | > loss_mel_ce: 3.3513975143432617 (3.656353071898633) + | > loss: 0.8473593592643738 (0.9233124821045585) + | > current_lr: 5e-06 + | > step_time: 0.3182 (0.31955128487213535) + | > loader_time: 0.0086 (0.009238407968528262) + + + --> TIME: 2024-10-20 08:38:50 -- STEP: 9622/16928 -- GLOBAL_STEP: 26550 + | > loss_text_ce: 0.03556976839900017 (0.036892687674202305) + | > loss_mel_ce: 3.4712557792663574 (3.656565445719774) + | > loss: 0.8767063617706299 (0.9233645326046556) + | > current_lr: 5e-06 + | > step_time: 0.2541 (0.3195738222514287) + | > loader_time: 0.0085 (0.009236387813990705) + + + --> TIME: 2024-10-20 08:39:17 -- STEP: 9672/16928 -- GLOBAL_STEP: 26600 + | > loss_text_ce: 0.03918030485510826 (0.03689182245937501) + | > loss_mel_ce: 3.60817551612854 (3.656689081165394) + | > loss: 0.911838948726654 (0.9233952251727469) + | > current_lr: 5e-06 + | > step_time: 0.3203 (0.31959085737110854) + | > loader_time: 0.0088 (0.009234934009925862) + + + --> TIME: 2024-10-20 08:39:44 -- STEP: 9722/16928 -- GLOBAL_STEP: 26650 + | > loss_text_ce: 0.0323297455906868 (0.03688440436711539) + | > loss_mel_ce: 3.3913910388946533 (3.6563978127014782) + | > loss: 0.8559302091598511 (0.9233205535423606) + | > current_lr: 5e-06 + | > step_time: 0.3406 (0.31958561626831405) + | > loader_time: 0.0083 (0.009233993994456369) + + + --> TIME: 2024-10-20 08:40:10 -- STEP: 9772/16928 -- GLOBAL_STEP: 26700 + | > loss_text_ce: 0.036086712032556534 (0.036882325078235405) + | > loss_mel_ce: 3.303746461868286 (3.6559630675829196) + | > loss: 0.8349583148956299 (0.9232113474383005) + | > current_lr: 5e-06 + | > step_time: 0.1364 (0.31953232065361614) + | > loader_time: 0.0081 (0.009233467631587871) + + + --> TIME: 2024-10-20 08:40:38 -- STEP: 9822/16928 -- GLOBAL_STEP: 26750 + | > loss_text_ce: 0.03638562932610512 (0.036876632682744034) + | > loss_mel_ce: 3.5559611320495605 (3.655581785253293) + | > loss: 0.898086667060852 (0.9231146037687815) + | > current_lr: 5e-06 + | > step_time: 0.4634 (0.31967489674726796) + | > loader_time: 0.0117 (0.009233672102802092) + + + --> TIME: 2024-10-20 08:41:06 -- STEP: 9872/16928 -- GLOBAL_STEP: 26800 + | > loss_text_ce: 0.036671798676252365 (0.036873317112303974) + | > loss_mel_ce: 4.038433074951172 (3.655654982650106) + | > loss: 1.0187761783599854 (0.9231320742028652) + | > current_lr: 5e-06 + | > step_time: 0.4072 (0.31970259527912753) + | > loader_time: 0.0103 (0.009234403143825514) + + + --> TIME: 2024-10-20 08:41:32 -- STEP: 9922/16928 -- GLOBAL_STEP: 26850 + | > loss_text_ce: 0.03529449552297592 (0.036868915292029854) + | > loss_mel_ce: 3.9682672023773193 (3.655561755991972) + | > loss: 1.0008903741836548 (0.9231076670964142) + | > current_lr: 5e-06 + | > step_time: 0.2684 (0.3196705621088061) + | > loader_time: 0.009 (0.009236611130015981) + + + --> TIME: 2024-10-20 08:41:58 -- STEP: 9972/16928 -- GLOBAL_STEP: 26900 + | > loss_text_ce: 0.042037233710289 (0.03686987869482608) + | > loss_mel_ce: 3.6200759410858154 (3.655436478916823) + | > loss: 0.9155282974243164 (0.9230765886888703) + | > current_lr: 5e-06 + | > step_time: 0.1713 (0.3196066541457528) + | > loader_time: 0.0079 (0.009239015118261869) + + + --> TIME: 2024-10-20 08:42:25 -- STEP: 10022/16928 -- GLOBAL_STEP: 26950 + | > loss_text_ce: 0.03866547718644142 (0.03686612572207252) + | > loss_mel_ce: 3.7612411975860596 (3.6555071727074777) + | > loss: 0.9499766826629639 (0.9230933239052714) + | > current_lr: 5e-06 + | > step_time: 0.2136 (0.3196120874488264) + | > loader_time: 0.0084 (0.00924054690402697) + + + --> TIME: 2024-10-20 08:42:52 -- STEP: 10072/16928 -- GLOBAL_STEP: 27000 + | > loss_text_ce: 0.036379653960466385 (0.0368627259140499) + | > loss_mel_ce: 3.6883175373077393 (3.6552518619205197) + | > loss: 0.9311742782592773 (0.9230286462603816) + | > current_lr: 5e-06 + | > step_time: 0.2727 (0.3196546555133543) + | > loader_time: 0.0101 (0.009242981743301215) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_27000.pth + + --> TIME: 2024-10-20 08:43:32 -- STEP: 10122/16928 -- GLOBAL_STEP: 27050 + | > loss_text_ce: 0.0358751155436039 (0.03685842164952406) + | > loss_mel_ce: 3.9188883304595947 (3.655300527051418) + | > loss: 0.9886908531188965 (0.9230397364422399) + | > current_lr: 5e-06 + | > step_time: 0.3082 (0.31972052056524186) + | > loader_time: 0.0089 (0.009244543717939617) + + + --> TIME: 2024-10-20 08:43:58 -- STEP: 10172/16928 -- GLOBAL_STEP: 27100 + | > loss_text_ce: 0.0381326861679554 (0.036855008491401775) + | > loss_mel_ce: 3.387270450592041 (3.655711415242455) + | > loss: 0.8563507795333862 (0.923141605217439) + | > current_lr: 5e-06 + | > step_time: 0.2711 (0.3196810499636433) + | > loader_time: 0.0106 (0.009244283704461382) + + + --> TIME: 2024-10-20 08:44:25 -- STEP: 10222/16928 -- GLOBAL_STEP: 27150 + | > loss_text_ce: 0.03847372904419899 (0.03685156172157836) + | > loss_mel_ce: 3.459516763687134 (3.6554181037222517) + | > loss: 0.874497652053833 (0.9230674156539013) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.31970728311350244) + | > loader_time: 0.0085 (0.009244835952843844) + + + --> TIME: 2024-10-20 08:44:53 -- STEP: 10272/16928 -- GLOBAL_STEP: 27200 + | > loss_text_ce: 0.037550557404756546 (0.03684759612385231) + | > loss_mel_ce: 3.809382915496826 (3.6550428121121503) + | > loss: 0.961733341217041 (0.9229726013422848) + | > current_lr: 5e-06 + | > step_time: 0.4162 (0.3197752809227443) + | > loader_time: 0.0187 (0.009246916716158177) + + + --> TIME: 2024-10-20 08:45:21 -- STEP: 10322/16928 -- GLOBAL_STEP: 27250 + | > loss_text_ce: 0.03487468510866165 (0.036845718987027654) + | > loss_mel_ce: 3.5167362689971924 (3.6551222023264516) + | > loss: 0.8879027366638184 (0.9229919796095317) + | > current_lr: 5e-06 + | > step_time: 0.4031 (0.31980951803411045) + | > loader_time: 0.009 (0.009247231774495492) + + + --> TIME: 2024-10-20 08:45:46 -- STEP: 10372/16928 -- GLOBAL_STEP: 27300 + | > loss_text_ce: 0.03831552341580391 (0.03684086353843831) + | > loss_mel_ce: 3.2808194160461426 (3.654756001165554) + | > loss: 0.8297837376594543 (0.9228992154652944) + | > current_lr: 5e-06 + | > step_time: 0.1377 (0.3197294426755101) + | > loader_time: 0.0075 (0.009246628876593018) + + + --> TIME: 2024-10-20 08:46:13 -- STEP: 10422/16928 -- GLOBAL_STEP: 27350 + | > loss_text_ce: 0.03629765287041664 (0.0368373919436731) + | > loss_mel_ce: 3.5039784908294678 (3.654922575736499) + | > loss: 0.8850690126419067 (0.9229399912120788) + | > current_lr: 5e-06 + | > step_time: 0.4452 (0.3197143425343935) + | > loader_time: 0.0105 (0.009248392112056957) + + + --> TIME: 2024-10-20 08:46:37 -- STEP: 10472/16928 -- GLOBAL_STEP: 27400 + | > loss_text_ce: 0.03686779364943504 (0.03683464001167657) + | > loss_mel_ce: 3.6816229820251465 (3.6547900919608254) + | > loss: 0.9296227097511292 (0.9229061822851621) + | > current_lr: 5e-06 + | > step_time: 0.1753 (0.3195582468196398) + | > loader_time: 0.0092 (0.00924795387900446) + + + --> TIME: 2024-10-20 08:47:06 -- STEP: 10522/16928 -- GLOBAL_STEP: 27450 + | > loss_text_ce: 0.03751208260655403 (0.036832328960674766) + | > loss_mel_ce: 4.025265216827393 (3.6548069156274394) + | > loss: 1.0156943798065186 (0.9229098104621675) + | > current_lr: 5e-06 + | > step_time: 0.3966 (0.31966920024518014) + | > loader_time: 0.0095 (0.009248895849979422) + + + --> TIME: 2024-10-20 08:47:33 -- STEP: 10572/16928 -- GLOBAL_STEP: 27500 + | > loss_text_ce: 0.03497153893113136 (0.036829309355301224) + | > loss_mel_ce: 3.7068445682525635 (3.655004576395463) + | > loss: 0.9354540109634399 (0.9229584707597689) + | > current_lr: 5e-06 + | > step_time: 0.2871 (0.31971399341169066) + | > loader_time: 0.0098 (0.009249218183171747) + + + --> TIME: 2024-10-20 08:47:59 -- STEP: 10622/16928 -- GLOBAL_STEP: 27550 + | > loss_text_ce: 0.03204982355237007 (0.036825303904883325) + | > loss_mel_ce: 3.493140459060669 (3.6548549745563643) + | > loss: 0.8812975883483887 (0.9229200689486473) + | > current_lr: 5e-06 + | > step_time: 0.4977 (0.3196815632983717) + | > loader_time: 0.0109 (0.009249473511443179) + + + --> TIME: 2024-10-20 08:48:26 -- STEP: 10672/16928 -- GLOBAL_STEP: 27600 + | > loss_text_ce: 0.041428063064813614 (0.03681920263431039) + | > loss_mel_ce: 3.885910749435425 (3.6547076787458903) + | > loss: 0.9818347096443176 (0.9228817197191215) + | > current_lr: 5e-06 + | > step_time: 0.2754 (0.3196629618910302) + | > loader_time: 0.0085 (0.00925066962145612) + + + --> TIME: 2024-10-20 08:48:53 -- STEP: 10722/16928 -- GLOBAL_STEP: 27650 + | > loss_text_ce: 0.038357608020305634 (0.03681458281243667) + | > loss_mel_ce: 3.4529101848602295 (3.6546374852700456) + | > loss: 0.8728169202804565 (0.9228630163813678) + | > current_lr: 5e-06 + | > step_time: 0.3591 (0.3196792464451082) + | > loader_time: 0.0086 (0.00925028508689235) + + + --> TIME: 2024-10-20 08:49:20 -- STEP: 10772/16928 -- GLOBAL_STEP: 27700 + | > loss_text_ce: 0.03917120769619942 (0.03681341801416242) + | > loss_mel_ce: 3.4293739795684814 (3.654734194654981) + | > loss: 0.8671362996101379 (0.9228869025487241) + | > current_lr: 5e-06 + | > step_time: 0.254 (0.31966378411704915) + | > loader_time: 0.0081 (0.009249459023905882) + + + --> TIME: 2024-10-20 08:49:45 -- STEP: 10822/16928 -- GLOBAL_STEP: 27750 + | > loss_text_ce: 0.03396141156554222 (0.03680917799264199) + | > loss_mel_ce: 3.4928367137908936 (3.654334969164549) + | > loss: 0.8816995024681091 (0.922786036170668) + | > current_lr: 5e-06 + | > step_time: 0.1908 (0.31959106401779436) + | > loader_time: 0.0092 (0.009248160518398751) + + + --> TIME: 2024-10-20 08:50:12 -- STEP: 10872/16928 -- GLOBAL_STEP: 27800 + | > loss_text_ce: 0.035433799028396606 (0.0368039496212161) + | > loss_mel_ce: 3.891436815261841 (3.6540098691179614) + | > loss: 0.9817176461219788 (0.9227034540859279) + | > current_lr: 5e-06 + | > step_time: 0.303 (0.3195795176264292) + | > loader_time: 0.0092 (0.009248623126916237) + + + --> TIME: 2024-10-20 08:50:38 -- STEP: 10922/16928 -- GLOBAL_STEP: 27850 + | > loss_text_ce: 0.038387175649404526 (0.03680153708305356) + | > loss_mel_ce: 3.0021018981933594 (3.6538069017464765) + | > loss: 0.7601222395896912 (0.9226521091176528) + | > current_lr: 5e-06 + | > step_time: 0.2276 (0.3195043756201733) + | > loader_time: 0.0087 (0.009248237822916287) + + + --> TIME: 2024-10-20 08:51:04 -- STEP: 10972/16928 -- GLOBAL_STEP: 27900 + | > loss_text_ce: 0.033211275935173035 (0.036796581712077704) + | > loss_mel_ce: 3.94303560256958 (3.653810987127466) + | > loss: 0.9940617084503174 (0.922651891611894) + | > current_lr: 5e-06 + | > step_time: 0.357 (0.3194599653568057) + | > loader_time: 0.0099 (0.009248468113606098) + + + --> TIME: 2024-10-20 08:51:31 -- STEP: 11022/16928 -- GLOBAL_STEP: 27950 + | > loss_text_ce: 0.041466839611530304 (0.0367943381140661) + | > loss_mel_ce: 3.9094247817993164 (3.6537628541990363) + | > loss: 0.9877229332923889 (0.9226392974677871) + | > current_lr: 5e-06 + | > step_time: 0.2562 (0.3194676304962393) + | > loader_time: 0.0084 (0.009249395260770139) + + + --> TIME: 2024-10-20 08:51:58 -- STEP: 11072/16928 -- GLOBAL_STEP: 28000 + | > loss_text_ce: 0.03668859601020813 (0.03679111416315695) + | > loss_mel_ce: 3.3173935413360596 (3.653927347068325) + | > loss: 0.8385205268859863 (0.9226796147009799) + | > current_lr: 5e-06 + | > step_time: 0.2459 (0.31947633477649323) + | > loader_time: 0.0096 (0.009251505911694737) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_28000.pth + + --> TIME: 2024-10-20 08:52:36 -- STEP: 11122/16928 -- GLOBAL_STEP: 28050 + | > loss_text_ce: 0.03743137791752815 (0.036787176702350356) + | > loss_mel_ce: 3.6104702949523926 (3.6537384092325444) + | > loss: 0.9119754433631897 (0.9226313958524307) + | > current_lr: 5e-06 + | > step_time: 0.4437 (0.319503708100967) + | > loader_time: 0.0114 (0.009253868223244113) + + + --> TIME: 2024-10-20 08:53:02 -- STEP: 11172/16928 -- GLOBAL_STEP: 28100 + | > loss_text_ce: 0.031097786501049995 (0.03678219148661101) + | > loss_mel_ce: 3.9996283054351807 (3.653486351172129) + | > loss: 1.0076814889907837 (0.9225671350145016) + | > current_lr: 5e-06 + | > step_time: 0.2735 (0.3194468972122124) + | > loader_time: 0.0099 (0.009253757276545236) + + + --> TIME: 2024-10-20 08:53:29 -- STEP: 11222/16928 -- GLOBAL_STEP: 28150 + | > loss_text_ce: 0.03436429053544998 (0.03677989341165233) + | > loss_mel_ce: 3.1383140087127686 (3.6527388011298063) + | > loss: 0.7931695580482483 (0.9223796729894059) + | > current_lr: 5e-06 + | > step_time: 0.4669 (0.3194070853049951) + | > loader_time: 0.0114 (0.009256989002822623) + + + --> TIME: 2024-10-20 08:53:57 -- STEP: 11272/16928 -- GLOBAL_STEP: 28200 + | > loss_text_ce: 0.034925397485494614 (0.03677323794952799) + | > loss_mel_ce: 3.0492289066314697 (3.6525238004028586) + | > loss: 0.7710385918617249 (0.9223242589478123) + | > current_lr: 5e-06 + | > step_time: 0.2701 (0.31946282210377164) + | > loader_time: 0.0104 (0.009258380039437058) + + + --> TIME: 2024-10-20 08:54:23 -- STEP: 11322/16928 -- GLOBAL_STEP: 28250 + | > loss_text_ce: 0.0390787273645401 (0.03677002663910515) + | > loss_mel_ce: 3.122899293899536 (3.652422360240139) + | > loss: 0.7904945015907288 (0.9222980960795577) + | > current_lr: 5e-06 + | > step_time: 0.3024 (0.31942918664510805) + | > loader_time: 0.0101 (0.00925918161142545) + + + --> TIME: 2024-10-20 08:54:51 -- STEP: 11372/16928 -- GLOBAL_STEP: 28300 + | > loss_text_ce: 0.0376247763633728 (0.03676855388630444) + | > loss_mel_ce: 3.61167311668396 (3.6521631018443714) + | > loss: 0.9123244881629944 (0.9222329133073922) + | > current_lr: 5e-06 + | > step_time: 0.3536 (0.3194954177463641) + | > loader_time: 0.0094 (0.009259443047429868) + + + --> TIME: 2024-10-20 08:55:17 -- STEP: 11422/16928 -- GLOBAL_STEP: 28350 + | > loss_text_ce: 0.03399066627025604 (0.03676469037460947) + | > loss_mel_ce: 3.2798056602478027 (3.651833939147359) + | > loss: 0.8284490704536438 (0.9221496567471894) + | > current_lr: 5e-06 + | > step_time: 0.4968 (0.31951006348883343) + | > loader_time: 0.0093 (0.009258088621360912) + + + --> TIME: 2024-10-20 08:55:46 -- STEP: 11472/16928 -- GLOBAL_STEP: 28400 + | > loss_text_ce: 0.03224782273173332 (0.03676016013673146) + | > loss_mel_ce: 3.5475707054138184 (3.6517278084445697) + | > loss: 0.8949546217918396 (0.9221219915216022) + | > current_lr: 5e-06 + | > step_time: 0.2516 (0.31960503443181976) + | > loader_time: 0.0093 (0.009257670850434514) + + + --> TIME: 2024-10-20 08:56:12 -- STEP: 11522/16928 -- GLOBAL_STEP: 28450 + | > loss_text_ce: 0.035744450986385345 (0.036758170660369786) + | > loss_mel_ce: 3.1318612098693848 (3.6512549057579604) + | > loss: 0.7919014096260071 (0.9220032684764935) + | > current_lr: 5e-06 + | > step_time: 0.3422 (0.3195847307843196) + | > loader_time: 0.0098 (0.009255944637556109) + + + --> TIME: 2024-10-20 08:56:39 -- STEP: 11572/16928 -- GLOBAL_STEP: 28500 + | > loss_text_ce: 0.0371217355132103 (0.03675263719195401) + | > loss_mel_ce: 3.443108081817627 (3.651223192856133) + | > loss: 0.8700574636459351 (0.9219939568969481) + | > current_lr: 5e-06 + | > step_time: 0.2511 (0.31957627589258464) + | > loader_time: 0.0116 (0.009254071339990564) + + + --> TIME: 2024-10-20 08:57:05 -- STEP: 11622/16928 -- GLOBAL_STEP: 28550 + | > loss_text_ce: 0.03635648638010025 (0.03674925080909382) + | > loss_mel_ce: 3.275191068649292 (3.65112564970347) + | > loss: 0.8278868794441223 (0.9219687245138704) + | > current_lr: 5e-06 + | > step_time: 0.17 (0.3195525801873136) + | > loader_time: 0.0089 (0.00925359490446611) + + + --> TIME: 2024-10-20 08:57:31 -- STEP: 11672/16928 -- GLOBAL_STEP: 28600 + | > loss_text_ce: 0.04010631516575813 (0.036745166634876425) + | > loss_mel_ce: 2.9139199256896973 (3.6508962019736675) + | > loss: 0.738506555557251 (0.9219103415303234) + | > current_lr: 5e-06 + | > step_time: 0.1173 (0.31950324163034965) + | > loader_time: 0.008 (0.009252177721184298) + + + --> TIME: 2024-10-20 08:57:59 -- STEP: 11722/16928 -- GLOBAL_STEP: 28650 + | > loss_text_ce: 0.03575028479099274 (0.03674254888492903) + | > loss_mel_ce: 3.597447156906128 (3.650573795595545) + | > loss: 0.9082993865013123 (0.9218290854977007) + | > current_lr: 5e-06 + | > step_time: 0.3088 (0.3195898023771972) + | > loader_time: 0.0088 (0.009251753556974404) + + + --> TIME: 2024-10-20 08:58:25 -- STEP: 11772/16928 -- GLOBAL_STEP: 28700 + | > loss_text_ce: 0.03552539646625519 (0.036738037436787815) + | > loss_mel_ce: 3.6944284439086914 (3.6502704134693853) + | > loss: 0.9324884414672852 (0.9217521120831143) + | > current_lr: 5e-06 + | > step_time: 0.185 (0.31952945263743077) + | > loader_time: 0.0084 (0.00924959617774171) + + + --> TIME: 2024-10-20 08:58:52 -- STEP: 11822/16928 -- GLOBAL_STEP: 28750 + | > loss_text_ce: 0.035356950014829636 (0.036734316140385505) + | > loss_mel_ce: 3.3683900833129883 (3.6499144394291654) + | > loss: 0.850936770439148 (0.9216621882458504) + | > current_lr: 5e-06 + | > step_time: 0.441 (0.31953960238647) + | > loader_time: 0.0092 (0.009250352637171977) + + + --> TIME: 2024-10-20 08:59:18 -- STEP: 11872/16928 -- GLOBAL_STEP: 28800 + | > loss_text_ce: 0.03555023670196533 (0.036730675690749615) + | > loss_mel_ce: 3.8689205646514893 (3.649762947363352) + | > loss: 0.976117730140686 (0.9216234051193188) + | > current_lr: 5e-06 + | > step_time: 0.2517 (0.31946657845032556) + | > loader_time: 0.0093 (0.009249254157440287) + + + --> TIME: 2024-10-20 08:59:45 -- STEP: 11922/16928 -- GLOBAL_STEP: 28850 + | > loss_text_ce: 0.034070227295160294 (0.0367262715830301) + | > loss_mel_ce: 3.2266345024108887 (3.6496978372840387) + | > loss: 0.8151761889457703 (0.9216060265761217) + | > current_lr: 5e-06 + | > step_time: 0.4432 (0.31951619120496944) + | > loader_time: 0.0086 (0.00924831049851447) + + + --> TIME: 2024-10-20 09:00:10 -- STEP: 11972/16928 -- GLOBAL_STEP: 28900 + | > loss_text_ce: 0.035678233951330185 (0.036726479652262084) + | > loss_mel_ce: 4.025249481201172 (3.6495671042064104) + | > loss: 1.015231966972351 (0.9215733953295764) + | > current_lr: 5e-06 + | > step_time: 0.2884 (0.31940522693369) + | > loader_time: 0.0087 (0.009247292713142273) + + + --> TIME: 2024-10-20 09:00:38 -- STEP: 12022/16928 -- GLOBAL_STEP: 28950 + | > loss_text_ce: 0.04623712599277496 (0.036724702588193735) + | > loss_mel_ce: 4.25438928604126 (3.649514157885899) + | > loss: 1.075156569480896 (0.9215597144737555) + | > current_lr: 5e-06 + | > step_time: 0.2496 (0.3194545416204495) + | > loader_time: 0.0093 (0.00924660328163948) + + + --> TIME: 2024-10-20 09:01:06 -- STEP: 12072/16928 -- GLOBAL_STEP: 29000 + | > loss_text_ce: 0.03425048664212227 (0.03672011360073075) + | > loss_mel_ce: 3.7699949741363525 (3.649300302177016) + | > loss: 0.9510613679885864 (0.921505103296322) + | > current_lr: 5e-06 + | > step_time: 0.2464 (0.31954211946589234) + | > loader_time: 0.0087 (0.009244893760851714) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_29000.pth + + --> TIME: 2024-10-20 09:01:44 -- STEP: 12122/16928 -- GLOBAL_STEP: 29050 + | > loss_text_ce: 0.03607287257909775 (0.03671650261097446) + | > loss_mel_ce: 3.315279722213745 (3.649044768071926) + | > loss: 0.8378381729125977 (0.9214403170295092) + | > current_lr: 5e-06 + | > step_time: 0.3141 (0.319550813808042) + | > loader_time: 0.0096 (0.00924652341843587) + + + --> TIME: 2024-10-20 09:02:11 -- STEP: 12172/16928 -- GLOBAL_STEP: 29100 + | > loss_text_ce: 0.03769025579094887 (0.036716389052038494) + | > loss_mel_ce: 3.5011775493621826 (3.6488131811812328) + | > loss: 0.8847169280052185 (0.9213823919336614) + | > current_lr: 5e-06 + | > step_time: 0.2243 (0.31955776963232435) + | > loader_time: 0.0093 (0.00924589277058097) + + + --> TIME: 2024-10-20 09:02:39 -- STEP: 12222/16928 -- GLOBAL_STEP: 29150 + | > loss_text_ce: 0.03363072872161865 (0.036713819888029756) + | > loss_mel_ce: 3.7632107734680176 (3.648472951722992) + | > loss: 0.9492104053497314 (0.9212966922837025) + | > current_lr: 5e-06 + | > step_time: 0.2266 (0.31959142389985734) + | > loader_time: 0.0083 (0.009246308993559655) + + + --> TIME: 2024-10-20 09:03:05 -- STEP: 12272/16928 -- GLOBAL_STEP: 29200 + | > loss_text_ce: 0.03676466643810272 (0.03671274807169358) + | > loss_mel_ce: 3.508347272872925 (3.648382780119058) + | > loss: 0.886277973651886 (0.9212738814553662) + | > current_lr: 5e-06 + | > step_time: 0.2754 (0.3195566199088503) + | > loader_time: 0.0099 (0.009246654177116065) + + + --> TIME: 2024-10-20 09:03:31 -- STEP: 12322/16928 -- GLOBAL_STEP: 29250 + | > loss_text_ce: 0.03486106917262077 (0.03671266939315679) + | > loss_mel_ce: 3.241523265838623 (3.6480502421699974) + | > loss: 0.8190960884094238 (0.9211907273104314) + | > current_lr: 5e-06 + | > step_time: 0.1903 (0.3194952432916957) + | > loader_time: 0.0088 (0.009246446776362915) + + + --> TIME: 2024-10-20 09:03:58 -- STEP: 12372/16928 -- GLOBAL_STEP: 29300 + | > loss_text_ce: 0.03621037304401398 (0.03670988477780964) + | > loss_mel_ce: 3.3788914680480957 (3.647694860890657) + | > loss: 0.8537754416465759 (0.921101185825894) + | > current_lr: 5e-06 + | > step_time: 0.2876 (0.31950156007659447) + | > loader_time: 0.0083 (0.009246813106259298) + + + --> TIME: 2024-10-20 09:04:24 -- STEP: 12422/16928 -- GLOBAL_STEP: 29350 + | > loss_text_ce: 0.03586069121956825 (0.03670692248166447) + | > loss_mel_ce: 3.6324851512908936 (3.647641221387934) + | > loss: 0.9170864820480347 (0.9210870353913772) + | > current_lr: 5e-06 + | > step_time: 0.309 (0.3194802030045294) + | > loader_time: 0.0097 (0.009245193709000447) + + + --> TIME: 2024-10-20 09:04:50 -- STEP: 12472/16928 -- GLOBAL_STEP: 29400 + | > loss_text_ce: 0.035151585936546326 (0.036702381377212885) + | > loss_mel_ce: 3.7448112964630127 (3.647273410429474) + | > loss: 0.9449906945228577 (0.9209939473815802) + | > current_lr: 5e-06 + | > step_time: 0.2719 (0.31942945313499604) + | > loader_time: 0.0086 (0.009243490586608031) + + + --> TIME: 2024-10-20 09:05:17 -- STEP: 12522/16928 -- GLOBAL_STEP: 29450 + | > loss_text_ce: 0.040208540856838226 (0.036699572224196696) + | > loss_mel_ce: 3.880974292755127 (3.6471484079849907) + | > loss: 0.980295717716217 (0.9209619945010673) + | > current_lr: 5e-06 + | > step_time: 0.228 (0.31944219206606084) + | > loader_time: 0.0083 (0.009242935143345645) + + + --> TIME: 2024-10-20 09:05:44 -- STEP: 12572/16928 -- GLOBAL_STEP: 29500 + | > loss_text_ce: 0.03770221024751663 (0.03669611467276684) + | > loss_mel_ce: 3.568053722381592 (3.6475070436365695) + | > loss: 0.9014390110969543 (0.9210507890486315) + | > current_lr: 5e-06 + | > step_time: 0.2468 (0.31944897144113155) + | > loader_time: 0.0088 (0.009242367827888441) + + + --> TIME: 2024-10-20 09:06:09 -- STEP: 12622/16928 -- GLOBAL_STEP: 29550 + | > loss_text_ce: 0.034498706459999084 (0.03669607571429933) + | > loss_mel_ce: 3.6436986923217773 (3.6470490509499234) + | > loss: 0.9195493459701538 (0.9209362811415873) + | > current_lr: 5e-06 + | > step_time: 0.4012 (0.31935235840491155) + | > loader_time: 0.01 (0.00924205230355653) + + + --> TIME: 2024-10-20 09:06:36 -- STEP: 12672/16928 -- GLOBAL_STEP: 29600 + | > loss_text_ce: 0.03531469404697418 (0.03669445982292251) + | > loss_mel_ce: 4.297616481781006 (3.647037814192549) + | > loss: 1.0832327604293823 (0.9209330679538349) + | > current_lr: 5e-06 + | > step_time: 0.3072 (0.3193713747190709) + | > loader_time: 0.0098 (0.00924167441524032) + + + --> TIME: 2024-10-20 09:07:03 -- STEP: 12722/16928 -- GLOBAL_STEP: 29650 + | > loss_text_ce: 0.0371098592877388 (0.03669186985517534) + | > loss_mel_ce: 4.351194858551025 (3.647025283375095) + | > loss: 1.097076177597046 (0.9209292877809261) + | > current_lr: 5e-06 + | > step_time: 0.431 (0.31938429123162276) + | > loader_time: 0.0092 (0.0092425933893962) + + + --> TIME: 2024-10-20 09:07:30 -- STEP: 12772/16928 -- GLOBAL_STEP: 29700 + | > loss_text_ce: 0.03220679983496666 (0.03668988260711494) + | > loss_mel_ce: 3.511202335357666 (3.646824635553614) + | > loss: 0.8858522772789001 (0.920878629003279) + | > current_lr: 5e-06 + | > step_time: 0.1857 (0.3194112440952117) + | > loader_time: 0.0102 (0.009243070631239211) + + + --> TIME: 2024-10-20 09:07:55 -- STEP: 12822/16928 -- GLOBAL_STEP: 29750 + | > loss_text_ce: 0.03823203593492508 (0.03668527119849508) + | > loss_mel_ce: 3.008272171020508 (3.6465588525237465) + | > loss: 0.7616260647773743 (0.920811030393717) + | > current_lr: 5e-06 + | > step_time: 0.3047 (0.31933343490448246) + | > loader_time: 0.0099 (0.009242129165953995) + + + --> TIME: 2024-10-20 09:08:22 -- STEP: 12872/16928 -- GLOBAL_STEP: 29800 + | > loss_text_ce: 0.036667488515377045 (0.03668007353694638) + | > loss_mel_ce: 3.1379716396331787 (3.646504864124425) + | > loss: 0.7936598062515259 (0.9207962338805125) + | > current_lr: 5e-06 + | > step_time: 0.2519 (0.3193268640171746) + | > loader_time: 0.0098 (0.009242603058693418) + + + --> TIME: 2024-10-20 09:08:50 -- STEP: 12922/16928 -- GLOBAL_STEP: 29850 + | > loss_text_ce: 0.036516427993774414 (0.03667839823933307) + | > loss_mel_ce: 3.4163057804107666 (3.646367162507563) + | > loss: 0.8632055521011353 (0.9207613896546838) + | > current_lr: 5e-06 + | > step_time: 0.261 (0.3193691716698221) + | > loader_time: 0.0094 (0.009243603351222745) + + + --> TIME: 2024-10-20 09:09:16 -- STEP: 12972/16928 -- GLOBAL_STEP: 29900 + | > loss_text_ce: 0.0358649305999279 (0.036675642470240744) + | > loss_mel_ce: 3.953559637069702 (3.646629477333883) + | > loss: 0.997356116771698 (0.9208262794011542) + | > current_lr: 5e-06 + | > step_time: 0.2089 (0.3193400901774729) + | > loader_time: 0.0079 (0.009243936771012807) + + + --> TIME: 2024-10-20 09:09:44 -- STEP: 13022/16928 -- GLOBAL_STEP: 29950 + | > loss_text_ce: 0.03305208310484886 (0.036672472030054824) + | > loss_mel_ce: 3.492633104324341 (3.6463272899842414) + | > loss: 0.8814212679862976 (0.9207499399563809) + | > current_lr: 5e-06 + | > step_time: 0.3606 (0.3194060712855927) + | > loader_time: 0.0086 (0.009243032548160566) + + + --> TIME: 2024-10-20 09:10:10 -- STEP: 13072/16928 -- GLOBAL_STEP: 30000 + | > loss_text_ce: 0.036871347576379776 (0.036671044772053095) + | > loss_mel_ce: 3.540825128555298 (3.6463624817505025) + | > loss: 0.8944241404533386 (0.9207583811029226) + | > current_lr: 5e-06 + | > step_time: 0.3459 (0.3193679298371111) + | > loader_time: 0.01 (0.009242820531822852) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_30000.pth + + --> TIME: 2024-10-20 09:10:51 -- STEP: 13122/16928 -- GLOBAL_STEP: 30050 + | > loss_text_ce: 0.04112861678004265 (0.03666801679572247) + | > loss_mel_ce: 4.220626354217529 (3.6460636724987476) + | > loss: 1.0654387474060059 (0.9206829217989767) + | > current_lr: 5e-06 + | > step_time: 0.4681 (0.31936799525996545) + | > loader_time: 0.0114 (0.009246301494744273) + + + --> TIME: 2024-10-20 09:11:17 -- STEP: 13172/16928 -- GLOBAL_STEP: 30100 + | > loss_text_ce: 0.033775828778743744 (0.036666237332284274) + | > loss_mel_ce: 3.1990933418273926 (3.645563698904586) + | > loss: 0.8082172870635986 (0.9205574835420125) + | > current_lr: 5e-06 + | > step_time: 0.2712 (0.3193616654385629) + | > loader_time: 0.0084 (0.009245832694760524) + + + --> TIME: 2024-10-20 09:11:44 -- STEP: 13222/16928 -- GLOBAL_STEP: 30150 + | > loss_text_ce: 0.04167329519987106 (0.03666552966070361) + | > loss_mel_ce: 3.7483019828796387 (3.6452583913236176) + | > loss: 0.9474937915802002 (0.9204809797313241) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.31933164802121944) + | > loader_time: 0.0077 (0.009245059382372687) + + + --> TIME: 2024-10-20 09:12:12 -- STEP: 13272/16928 -- GLOBAL_STEP: 30200 + | > loss_text_ce: 0.0343034490942955 (0.036661109708967014) + | > loss_mel_ce: 3.64170503616333 (3.644995732433901) + | > loss: 0.919002115726471 (0.9204142100350048) + | > current_lr: 5e-06 + | > step_time: 0.2889 (0.31939730362550095) + | > loader_time: 0.01 (0.009244721200970543) + + + --> TIME: 2024-10-20 09:12:37 -- STEP: 13322/16928 -- GLOBAL_STEP: 30250 + | > loss_text_ce: 0.03582879528403282 (0.036658707644016984) + | > loss_mel_ce: 3.8561251163482666 (3.644995584446395) + | > loss: 0.972988486289978 (0.9204135725375071) + | > current_lr: 5e-06 + | > step_time: 0.3061 (0.31929971731169) + | > loader_time: 0.0084 (0.009245016842950728) + + + --> TIME: 2024-10-20 09:13:03 -- STEP: 13372/16928 -- GLOBAL_STEP: 30300 + | > loss_text_ce: 0.035138051956892014 (0.0366557279402224) + | > loss_mel_ce: 4.632015228271484 (3.644792751855134) + | > loss: 1.1667883396148682 (0.9203621194751684) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.319266633665779) + | > loader_time: 0.009 (0.009244465325144674) + + + --> TIME: 2024-10-20 09:13:30 -- STEP: 13422/16928 -- GLOBAL_STEP: 30350 + | > loss_text_ce: 0.038160327821969986 (0.03665342437073699) + | > loss_mel_ce: 2.9800519943237305 (3.6445231862935734) + | > loss: 0.7545530796051025 (0.9202941521911183) + | > current_lr: 5e-06 + | > step_time: 0.252 (0.31927329856936204) + | > loader_time: 0.0092 (0.009244877488118202) + + + --> TIME: 2024-10-20 09:13:55 -- STEP: 13472/16928 -- GLOBAL_STEP: 30400 + | > loss_text_ce: 0.03581229969859123 (0.03665256624097879) + | > loss_mel_ce: 3.1110126972198486 (3.6442594898780563) + | > loss: 0.786706268787384 (0.9202280135519997) + | > current_lr: 5e-06 + | > step_time: 0.1741 (0.31919289723640365) + | > loader_time: 0.0094 (0.009244984533469605) + + + --> TIME: 2024-10-20 09:14:21 -- STEP: 13522/16928 -- GLOBAL_STEP: 30450 + | > loss_text_ce: 0.0335029773414135 (0.0366502006693456) + | > loss_mel_ce: 3.7435498237609863 (3.644065315792889) + | > loss: 0.944263219833374 (0.9201788786485199) + | > current_lr: 5e-06 + | > step_time: 0.4632 (0.3191611962000339) + | > loader_time: 0.0098 (0.009245340860448889) + + + --> TIME: 2024-10-20 09:14:48 -- STEP: 13572/16928 -- GLOBAL_STEP: 30500 + | > loss_text_ce: 0.03744233399629593 (0.03664739548582492) + | > loss_mel_ce: 3.6503589153289795 (3.6439754894410057) + | > loss: 0.9219503402709961 (0.9201557207583608) + | > current_lr: 5e-06 + | > step_time: 0.3049 (0.3191867690328084) + | > loader_time: 0.0085 (0.00924488945130993) + + + --> TIME: 2024-10-20 09:15:16 -- STEP: 13622/16928 -- GLOBAL_STEP: 30550 + | > loss_text_ce: 0.03257627785205841 (0.03664451009249185) + | > loss_mel_ce: 3.6047136783599854 (3.643778359114625) + | > loss: 0.9093225002288818 (0.9201057168412115) + | > current_lr: 5e-06 + | > step_time: 0.2899 (0.3192103267713642) + | > loader_time: 0.0087 (0.009245240133495803) + + + --> TIME: 2024-10-20 09:15:42 -- STEP: 13672/16928 -- GLOBAL_STEP: 30600 + | > loss_text_ce: 0.03477403149008751 (0.03664064711622449) + | > loss_mel_ce: 3.7678489685058594 (3.643789247855448) + | > loss: 0.9506557583808899 (0.920107473283047) + | > current_lr: 5e-06 + | > step_time: 0.2549 (0.3191725638367543) + | > loader_time: 0.0083 (0.009243724585416486) + + + --> TIME: 2024-10-20 09:16:07 -- STEP: 13722/16928 -- GLOBAL_STEP: 30650 + | > loss_text_ce: 0.03373441845178604 (0.03663906749626219) + | > loss_mel_ce: 3.6975772380828857 (3.643469408517189) + | > loss: 0.932827889919281 (0.9200271185503255) + | > current_lr: 5e-06 + | > step_time: 0.3026 (0.3191064271265241) + | > loader_time: 0.0098 (0.009242740669189304) + + + --> TIME: 2024-10-20 09:16:34 -- STEP: 13772/16928 -- GLOBAL_STEP: 30700 + | > loss_text_ce: 0.03617032617330551 (0.03663690713678661) + | > loss_mel_ce: 3.4468560218811035 (3.643555506574806) + | > loss: 0.8707565665245056 (0.9200481029745108) + | > current_lr: 5e-06 + | > step_time: 0.3541 (0.3191394745663019) + | > loader_time: 0.0089 (0.009242570421012215) + + + --> TIME: 2024-10-20 09:17:01 -- STEP: 13822/16928 -- GLOBAL_STEP: 30750 + | > loss_text_ce: 0.03735005855560303 (0.0366358752655531) + | > loss_mel_ce: 4.404875755310059 (3.6432953428176806) + | > loss: 1.1105564832687378 (0.9199828040692969) + | > current_lr: 5e-06 + | > step_time: 0.4003 (0.3191375257723617) + | > loader_time: 0.0089 (0.009242005983666443) + + + --> TIME: 2024-10-20 09:17:28 -- STEP: 13872/16928 -- GLOBAL_STEP: 30800 + | > loss_text_ce: 0.033396072685718536 (0.036631482351203885) + | > loss_mel_ce: 4.054239273071289 (3.6431220016574364) + | > loss: 1.0219088792800903 (0.9199383705553643) + | > current_lr: 5e-06 + | > step_time: 0.1709 (0.3191465236152055) + | > loader_time: 0.0084 (0.00924140053361893) + + + --> TIME: 2024-10-20 09:17:55 -- STEP: 13922/16928 -- GLOBAL_STEP: 30850 + | > loss_text_ce: 0.0348580963909626 (0.036628478597632824) + | > loss_mel_ce: 3.709744453430176 (3.6433400299302154) + | > loss: 0.9361506104469299 (0.9199921266860686) + | > current_lr: 5e-06 + | > step_time: 0.4991 (0.31915188688533824) + | > loader_time: 0.0105 (0.009241005004676742) + + + --> TIME: 2024-10-20 09:18:21 -- STEP: 13972/16928 -- GLOBAL_STEP: 30900 + | > loss_text_ce: 0.03843618184328079 (0.03662571694590991) + | > loss_mel_ce: 3.731858968734741 (3.643165671804932) + | > loss: 0.9425737857818604 (0.9199478467342473) + | > current_lr: 5e-06 + | > step_time: 0.2868 (0.3191100684885918) + | > loader_time: 0.0087 (0.009241355001295035) + + + --> TIME: 2024-10-20 09:18:46 -- STEP: 14022/16928 -- GLOBAL_STEP: 30950 + | > loss_text_ce: 0.03529234603047371 (0.036622943433413144) + | > loss_mel_ce: 3.2203283309936523 (3.6426367705974543) + | > loss: 0.8139051795005798 (0.9198149280632432) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.3190514679863726) + | > loader_time: 0.0084 (0.009241011101583001) + + + --> TIME: 2024-10-20 09:19:14 -- STEP: 14072/16928 -- GLOBAL_STEP: 31000 + | > loss_text_ce: 0.03398489952087402 (0.03662082514206221) + | > loss_mel_ce: 3.7326643466949463 (3.6426315724748046) + | > loss: 0.9416623115539551 (0.91981309897575) + | > current_lr: 5e-06 + | > step_time: 0.2867 (0.31909601192978565) + | > loader_time: 0.0098 (0.00924304876563356) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_31000.pth + + --> TIME: 2024-10-20 09:19:52 -- STEP: 14122/16928 -- GLOBAL_STEP: 31050 + | > loss_text_ce: 0.030492538586258888 (0.03661804127253272) + | > loss_mel_ce: 3.2045717239379883 (3.6427559181678535) + | > loss: 0.8087660670280457 (0.9198434894178799) + | > current_lr: 5e-06 + | > step_time: 0.1885 (0.319099943811065) + | > loader_time: 0.0099 (0.009245609328828152) + + + --> TIME: 2024-10-20 09:20:19 -- STEP: 14172/16928 -- GLOBAL_STEP: 31100 + | > loss_text_ce: 0.03143855929374695 (0.036615400813912125) + | > loss_mel_ce: 3.9349417686462402 (3.6427875113561266) + | > loss: 0.9915950894355774 (0.9198507275952816) + | > current_lr: 5e-06 + | > step_time: 0.2026 (0.31910421220454777) + | > loader_time: 0.0087 (0.009246413414336296) + + + --> TIME: 2024-10-20 09:20:46 -- STEP: 14222/16928 -- GLOBAL_STEP: 31150 + | > loss_text_ce: 0.03860295191407204 (0.03661442602847218) + | > loss_mel_ce: 3.686943769454956 (3.642642136482461) + | > loss: 0.9313867092132568 (0.9198141401829288) + | > current_lr: 5e-06 + | > step_time: 0.4312 (0.31911519456892085) + | > loader_time: 0.0103 (0.009246356895311574) + + + --> TIME: 2024-10-20 09:21:12 -- STEP: 14272/16928 -- GLOBAL_STEP: 31200 + | > loss_text_ce: 0.03536217287182808 (0.03661111801914396) + | > loss_mel_ce: 3.686626672744751 (3.642424663603974) + | > loss: 0.9304972290992737 (0.9197589449642953) + | > current_lr: 5e-06 + | > step_time: 0.3025 (0.31907029084092814) + | > loader_time: 0.0086 (0.0092456198961478) + + + --> TIME: 2024-10-20 09:21:39 -- STEP: 14322/16928 -- GLOBAL_STEP: 31250 + | > loss_text_ce: 0.031861312687397 (0.03660786877214727) + | > loss_mel_ce: 3.4683167934417725 (3.64245281930643) + | > loss: 0.8750445246696472 (0.9197651715713778) + | > current_lr: 5e-06 + | > step_time: 0.4422 (0.3191018724321668) + | > loader_time: 0.0089 (0.00924504247728529) + + + --> TIME: 2024-10-20 09:22:06 -- STEP: 14372/16928 -- GLOBAL_STEP: 31300 + | > loss_text_ce: 0.034856222569942474 (0.03660508962513992) + | > loss_mel_ce: 4.596295356750488 (3.6424305241916826) + | > loss: 1.1577879190444946 (0.9197589029990098) + | > current_lr: 5e-06 + | > step_time: 0.3347 (0.31911584140197347) + | > loader_time: 0.0105 (0.009245718882470734) + + + --> TIME: 2024-10-20 09:22:32 -- STEP: 14422/16928 -- GLOBAL_STEP: 31350 + | > loss_text_ce: 0.03635541349649429 (0.03660128504875283) + | > loss_mel_ce: 3.8236491680145264 (3.642414309300359) + | > loss: 0.9650011658668518 (0.9197538981257819) + | > current_lr: 5e-06 + | > step_time: 0.3807 (0.3191069739980118) + | > loader_time: 0.0085 (0.00924488690074347) + + + --> TIME: 2024-10-20 09:23:01 -- STEP: 14472/16928 -- GLOBAL_STEP: 31400 + | > loss_text_ce: 0.03387190401554108 (0.036599317774451495) + | > loss_mel_ce: 3.6902520656585693 (3.642398518330488) + | > loss: 0.9310309886932373 (0.9197494585485073) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.31917564173045226) + | > loader_time: 0.0078 (0.00924372401205855) + + + --> TIME: 2024-10-20 09:23:27 -- STEP: 14522/16928 -- GLOBAL_STEP: 31450 + | > loss_text_ce: 0.037135086953639984 (0.0365980174486094) + | > loss_mel_ce: 3.175920248031616 (3.6421456274915998) + | > loss: 0.8032638430595398 (0.9196859107638437) + | > current_lr: 5e-06 + | > step_time: 0.3056 (0.319159124019372) + | > loader_time: 0.0091 (0.009242548625374953) + + + --> TIME: 2024-10-20 09:23:53 -- STEP: 14572/16928 -- GLOBAL_STEP: 31500 + | > loss_text_ce: 0.03415786847472191 (0.03659583889027212) + | > loss_mel_ce: 3.095369815826416 (3.6419275923479466) + | > loss: 0.7823818922042847 (0.9196308573425193) + | > current_lr: 5e-06 + | > step_time: 0.1192 (0.319127377215696) + | > loader_time: 0.0076 (0.009241385346024417) + + + --> TIME: 2024-10-20 09:24:19 -- STEP: 14622/16928 -- GLOBAL_STEP: 31550 + | > loss_text_ce: 0.03736137971282005 (0.03659227705670047) + | > loss_mel_ce: 3.6242668628692627 (3.641902083754458) + | > loss: 0.9154070615768433 (0.9196235897431473) + | > current_lr: 5e-06 + | > step_time: 0.2549 (0.3190641982857691) + | > loader_time: 0.009 (0.009239343591793618) + + + --> TIME: 2024-10-20 09:24:46 -- STEP: 14672/16928 -- GLOBAL_STEP: 31600 + | > loss_text_ce: 0.034960564225912094 (0.036590687638212625) + | > loss_mel_ce: 3.6410086154937744 (3.6421400954055163) + | > loss: 0.918992280960083 (0.9196826953038719) + | > current_lr: 5e-06 + | > step_time: 0.2035 (0.3191161090051073) + | > loader_time: 0.0087 (0.009238062404104234) + + + --> TIME: 2024-10-20 09:25:14 -- STEP: 14722/16928 -- GLOBAL_STEP: 31650 + | > loss_text_ce: 0.039157766848802567 (0.03658759203772231) + | > loss_mel_ce: 3.5885989665985107 (3.642118961991475) + | > loss: 0.9069392085075378 (0.9196766380616916) + | > current_lr: 5e-06 + | > step_time: 0.3776 (0.3191437250957946) + | > loader_time: 0.0091 (0.009236850827272406) + + + --> TIME: 2024-10-20 09:25:39 -- STEP: 14772/16928 -- GLOBAL_STEP: 31700 + | > loss_text_ce: 0.037193793803453445 (0.03658481212869998) + | > loss_mel_ce: 3.7327685356140137 (3.6418245483019343) + | > loss: 0.9424905776977539 (0.9196023396641266) + | > current_lr: 5e-06 + | > step_time: 0.2708 (0.31909355208895035) + | > loader_time: 0.0089 (0.00923532108740207) + + + --> TIME: 2024-10-20 09:26:06 -- STEP: 14822/16928 -- GLOBAL_STEP: 31750 + | > loss_text_ce: 0.03484920412302017 (0.03658178666320775) + | > loss_mel_ce: 3.557533025741577 (3.6416797675901122) + | > loss: 0.8980955481529236 (0.919565388140867) + | > current_lr: 5e-06 + | > step_time: 0.4454 (0.31909495728039367) + | > loader_time: 0.0109 (0.009234668690253309) + + + --> TIME: 2024-10-20 09:26:33 -- STEP: 14872/16928 -- GLOBAL_STEP: 31800 + | > loss_text_ce: 0.034332286566495895 (0.03657846139443834) + | > loss_mel_ce: 3.9349451065063477 (3.6418427103562276) + | > loss: 0.9923193454742432 (0.9196052925040993) + | > current_lr: 5e-06 + | > step_time: 0.3734 (0.31910544646690436) + | > loader_time: 0.0114 (0.00923532209068038) + + + --> TIME: 2024-10-20 09:27:00 -- STEP: 14922/16928 -- GLOBAL_STEP: 31850 + | > loss_text_ce: 0.035663604736328125 (0.03657463186985583) + | > loss_mel_ce: 3.4938342571258545 (3.6418161027761586) + | > loss: 0.8823744654655457 (0.9195976832399994) + | > current_lr: 5e-06 + | > step_time: 0.3126 (0.31910450618942254) + | > loader_time: 0.0096 (0.009236215474602029) + + + --> TIME: 2024-10-20 09:27:27 -- STEP: 14972/16928 -- GLOBAL_STEP: 31900 + | > loss_text_ce: 0.03614061325788498 (0.03657070637675363) + | > loss_mel_ce: 3.7828099727630615 (3.641814408727804) + | > loss: 0.954737663269043 (0.9195962783537412) + | > current_lr: 5e-06 + | > step_time: 0.2583 (0.31909416045475014) + | > loader_time: 0.0097 (0.009237218868149606) + + + --> TIME: 2024-10-20 09:27:52 -- STEP: 15022/16928 -- GLOBAL_STEP: 31950 + | > loss_text_ce: 0.03668217733502388 (0.036566600871224375) + | > loss_mel_ce: 3.284248113632202 (3.641627921275333) + | > loss: 0.830232560634613 (0.9195486301192429) + | > current_lr: 5e-06 + | > step_time: 0.4625 (0.31903936461084376) + | > loader_time: 0.0086 (0.009237292396642907) + + + --> TIME: 2024-10-20 09:28:18 -- STEP: 15072/16928 -- GLOBAL_STEP: 32000 + | > loss_text_ce: 0.034827426075935364 (0.03656487745139449) + | > loss_mel_ce: 3.728034019470215 (3.6416603829039382) + | > loss: 0.9407153725624084 (0.9195563146832643) + | > current_lr: 5e-06 + | > step_time: 0.3028 (0.31901779248835915) + | > loader_time: 0.0101 (0.00923853943365646) + + + > CHECKPOINT : /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b/checkpoint_32000.pth + + --> TIME: 2024-10-20 09:28:55 -- STEP: 15122/16928 -- GLOBAL_STEP: 32050 + | > loss_text_ce: 0.03646054491400719 (0.03656242652814067) + | > loss_mel_ce: 4.092257976531982 (3.641404828749673) + | > loss: 1.032179594039917 (0.9194918134046632) + | > current_lr: 5e-06 + | > step_time: 0.2517 (0.318988219106533) + | > loader_time: 0.0084 (0.00924102329195624) + + + --> TIME: 2024-10-20 09:29:21 -- STEP: 15172/16928 -- GLOBAL_STEP: 32100 + | > loss_text_ce: 0.0392875000834465 (0.036559671472567494) + | > loss_mel_ce: 3.3480472564697266 (3.641296645628881) + | > loss: 0.8468337059020996 (0.9194640788661438) + | > current_lr: 5e-06 + | > step_time: 0.2887 (0.31897749586215957) + | > loader_time: 0.01 (0.009242297538880005) + + + --> TIME: 2024-10-20 09:29:49 -- STEP: 15222/16928 -- GLOBAL_STEP: 32150 + | > loss_text_ce: 0.03526229038834572 (0.03655778229674179) + | > loss_mel_ce: 3.1749064922332764 (3.6410604003004825) + | > loss: 0.8025422096252441 (0.9194045452568193) + | > current_lr: 5e-06 + | > step_time: 0.2283 (0.31901363967173246) + | > loader_time: 0.0086 (0.009244014126385812) + + + --> TIME: 2024-10-20 09:30:16 -- STEP: 15272/16928 -- GLOBAL_STEP: 32200 + | > loss_text_ce: 0.03894428536295891 (0.03655560425460246) + | > loss_mel_ce: 3.4370663166046143 (3.640877902991738) + | > loss: 0.869002640247345 (0.9193583764194381) + | > current_lr: 5e-06 + | > step_time: 0.224 (0.31903384882720603) + | > loader_time: 0.0084 (0.009244080595847743) + + + --> TIME: 2024-10-20 09:30:44 -- STEP: 15322/16928 -- GLOBAL_STEP: 32250 + | > loss_text_ce: 0.03797472268342972 (0.03655255061482252) + | > loss_mel_ce: 3.941019296646118 (3.6408679821739898) + | > loss: 0.9947485327720642 (0.9193551328206806) + | > current_lr: 5e-06 + | > step_time: 0.3471 (0.31909709849069406) + | > loader_time: 0.009 (0.009245082333386355) + + + --> TIME: 2024-10-20 09:31:10 -- STEP: 15372/16928 -- GLOBAL_STEP: 32300 + | > loss_text_ce: 0.03488818556070328 (0.036549650686287724) + | > loss_mel_ce: 4.058953762054443 (3.640572224949413) + | > loss: 1.0234605073928833 (0.9192804685373838) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.3190733403269895) + | > loader_time: 0.0093 (0.009244336627158138) + + + --> TIME: 2024-10-20 09:31:37 -- STEP: 15422/16928 -- GLOBAL_STEP: 32350 + | > loss_text_ce: 0.03503616154193878 (0.03654600458958568) + | > loss_mel_ce: 3.319627285003662 (3.64016904547712) + | > loss: 0.8386658430099487 (0.9191787621496006) + | > current_lr: 5e-06 + | > step_time: 0.2326 (0.31909499872118824) + | > loader_time: 0.0082 (0.009244958956469465) + + + --> TIME: 2024-10-20 09:32:06 -- STEP: 15472/16928 -- GLOBAL_STEP: 32400 + | > loss_text_ce: 0.03702346980571747 (0.03654411957106111) + | > loss_mel_ce: 3.249236583709717 (3.640229930321088) + | > loss: 0.8215650320053101 (0.9191935121157855) + | > current_lr: 5e-06 + | > step_time: 0.2871 (0.31915417242691196) + | > loader_time: 0.0084 (0.00924424102843854) + + + --> TIME: 2024-10-20 09:32:33 -- STEP: 15522/16928 -- GLOBAL_STEP: 32450 + | > loss_text_ce: 0.03526970371603966 (0.03653936568662501) + | > loss_mel_ce: 3.625678062438965 (3.639987122504125) + | > loss: 0.9152369499206543 (0.9191316216819265) + | > current_lr: 5e-06 + | > step_time: 0.3594 (0.31917821522334094) + | > loader_time: 0.0098 (0.009244856446109732) + + + --> TIME: 2024-10-20 09:32:59 -- STEP: 15572/16928 -- GLOBAL_STEP: 32500 + | > loss_text_ce: 0.0347650982439518 (0.036536441890787405) + | > loss_mel_ce: 3.627202033996582 (3.6399671798832705) + | > loss: 0.9154917597770691 (0.9191259050925042) + | > current_lr: 5e-06 + | > step_time: 0.2545 (0.319135672722137) + | > loader_time: 0.0088 (0.009244235750644685) + + + --> TIME: 2024-10-20 09:33:26 -- STEP: 15622/16928 -- GLOBAL_STEP: 32550 + | > loss_text_ce: 0.03697595372796059 (0.036532735426659455) + | > loss_mel_ce: 3.042867422103882 (3.6395293891910647) + | > loss: 0.7699608206748962 (0.9190155308044547) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.31914509095145666) + | > loader_time: 0.0084 (0.009243273091001815) + + + --> TIME: 2024-10-20 09:33:52 -- STEP: 15672/16928 -- GLOBAL_STEP: 32600 + | > loss_text_ce: 0.03580746799707413 (0.0365277155046458) + | > loss_mel_ce: 3.167797565460205 (3.63957975843599) + | > loss: 0.8009012341499329 (0.9190268681411721) + | > current_lr: 5e-06 + | > step_time: 0.251 (0.31914213042651085) + | > loader_time: 0.0103 (0.009243667201767029) + + + --> TIME: 2024-10-20 09:34:19 -- STEP: 15722/16928 -- GLOBAL_STEP: 32650 + | > loss_text_ce: 0.032732076942920685 (0.036524918867373905) + | > loss_mel_ce: 3.679931402206421 (3.6395043880335023) + | > loss: 0.928165853023529 (0.919007326378772) + | > current_lr: 5e-06 + | > step_time: 0.2885 (0.319109786998951) + | > loader_time: 0.0089 (0.009242937728931318) + + + --> TIME: 2024-10-20 09:34:46 -- STEP: 15772/16928 -- GLOBAL_STEP: 32700 + | > loss_text_ce: 0.035341452807188034 (0.03652192492899182) + | > loss_mel_ce: 3.7680327892303467 (3.639307594574438) + | > loss: 0.9508435726165771 (0.9189573795306268) + | > current_lr: 5e-06 + | > step_time: 0.2716 (0.3191377381488751) + | > loader_time: 0.0088 (0.009244011163771998) + + + --> TIME: 2024-10-20 09:35:12 -- STEP: 15822/16928 -- GLOBAL_STEP: 32750 + | > loss_text_ce: 0.035810597240924835 (0.03651842825557842) + | > loss_mel_ce: 3.7181591987609863 (3.639304540353514) + | > loss: 0.938492476940155 (0.9189557418059555) + | > current_lr: 5e-06 + | > step_time: 0.4272 (0.31911804559724744) + | > loader_time: 0.0092 (0.009243409773173561) + + + --> TIME: 2024-10-20 09:35:37 -- STEP: 15872/16928 -- GLOBAL_STEP: 32800 + | > loss_text_ce: 0.033837273716926575 (0.03651285780436027) + | > loss_mel_ce: 3.0359513759613037 (3.639155644142339) + | > loss: 0.7674471735954285 (0.9189171251247547) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.31902708979924344) + | > loader_time: 0.0081 (0.00924207086885166) + + ! Run is kept in /kaggle/temp/checkpoints/GPT_XTTS_FT-October-20-2024_04+25AM-1a53d9b