diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,18172 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: False + | > Precision: float32 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 12 + | > Num. of Torch Threads: 1 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: False + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b + + > Model has 522156884 parameters + + > EPOCH: 0/5 + --> checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b + + > TRAINING (2024-10-19 22:48:01)  + + --> TIME: 2024-10-19 22:48:08 -- STEP: 0/69059 -- GLOBAL_STEP: 0 + | > loss_text_ce: 0.2523742914199829 (0.2523742914199829) + | > loss_mel_ce: 5.576013088226318 (5.576013088226318) + | > loss: 1.457096815109253 (1.457096815109253) + | > current_lr: 5e-06 + | > step_time: 1.5146 (1.5145678520202637) + | > loader_time: 5.55 (5.54999041557312) + + + --> TIME: 2024-10-19 22:48:28 -- STEP: 50/69059 -- GLOBAL_STEP: 50 + | > loss_text_ce: 0.24215096235275269 (0.2523053967952728) + | > loss_mel_ce: 4.890015602111816 (5.050488328933715) + | > loss: 1.2830415964126587 (1.3256984329223633) + | > current_lr: 5e-06 + | > step_time: 0.3405 (0.2539503002166748) + | > loader_time: 0.0197 (0.02737452507019043) + + + --> TIME: 2024-10-19 22:48:47 -- STEP: 100/69059 -- GLOBAL_STEP: 100 + | > loss_text_ce: 0.22201569378376007 (0.24012222588062287) + | > loss_mel_ce: 4.439034461975098 (4.861174302101134) + | > loss: 1.1652625799179077 (1.2753241336345673) + | > current_lr: 5e-06 + | > step_time: 0.1979 (0.24893883228302002) + | > loader_time: 0.0154 (0.021878082752227784) + + + --> TIME: 2024-10-19 22:49:06 -- STEP: 150/69059 -- GLOBAL_STEP: 150 + | > loss_text_ce: 0.20445945858955383 (0.2306542177995046) + | > loss_mel_ce: 4.539186477661133 (4.7399575201670325) + | > loss: 1.1859115362167358 (1.2426529335975647) + | > current_lr: 5e-06 + | > step_time: 0.1801 (0.24901590983072916) + | > loader_time: 0.0155 (0.01973138173421224) + + + --> TIME: 2024-10-19 22:49:25 -- STEP: 200/69059 -- GLOBAL_STEP: 200 + | > loss_text_ce: 0.18592597544193268 (0.22219832926988603) + | > loss_mel_ce: 4.597056865692139 (4.658837420940396) + | > loss: 1.1957457065582275 (1.2202589350938795) + | > current_lr: 5e-06 + | > step_time: 0.2079 (0.2511735010147094) + | > loader_time: 0.0179 (0.01849950194358825) + + + --> TIME: 2024-10-19 22:49:44 -- STEP: 250/69059 -- GLOBAL_STEP: 250 + | > loss_text_ce: 0.1510399729013443 (0.2144932256937027) + | > loss_mel_ce: 4.400592803955078 (4.606624824523922) + | > loss: 1.1379082202911377 (1.2052795119285582) + | > current_lr: 5e-06 + | > step_time: 0.5422 (0.24842801475524895) + | > loader_time: 0.0207 (0.01759616374969481) + + + --> TIME: 2024-10-19 22:50:03 -- STEP: 300/69059 -- GLOBAL_STEP: 300 + | > loss_text_ce: 0.16636858880519867 (0.20692162980635964) + | > loss_mel_ce: 4.273943901062012 (4.555460578600562) + | > loss: 1.1100780963897705 (1.1905955509344741) + | > current_lr: 5e-06 + | > step_time: 0.1818 (0.24934062401453644) + | > loader_time: 0.0107 (0.016994080543518055) + + + --> TIME: 2024-10-19 22:50:21 -- STEP: 350/69059 -- GLOBAL_STEP: 350 + | > loss_text_ce: 0.1602575033903122 (0.19997937875134605) + | > loss_mel_ce: 4.368553161621094 (4.5200816413334435) + | > loss: 1.1322026252746582 (1.1800152533394948) + | > current_lr: 5e-06 + | > step_time: 0.1795 (0.24810055664607453) + | > loader_time: 0.0111 (0.01645203181675502) + + + --> TIME: 2024-10-19 22:50:40 -- STEP: 400/69059 -- GLOBAL_STEP: 400 + | > loss_text_ce: 0.12408541142940521 (0.19291188307106494) + | > loss_mel_ce: 4.223232746124268 (4.487980144619941) + | > loss: 1.0868295431137085 (1.1702230057120315) + | > current_lr: 5e-06 + | > step_time: 0.4417 (0.24857359707355495) + | > loader_time: 0.0112 (0.016055732965469343) + + + --> TIME: 2024-10-19 22:50:58 -- STEP: 450/69059 -- GLOBAL_STEP: 450 + | > loss_text_ce: 0.12150727957487106 (0.18605832522114119) + | > loss_mel_ce: 3.851632833480835 (4.455424908002219) + | > loss: 0.9932850003242493 (1.1603708077801593) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.2465080584420098) + | > loader_time: 0.0129 (0.015726700888739693) + + + --> TIME: 2024-10-19 22:51:16 -- STEP: 500/69059 -- GLOBAL_STEP: 500 + | > loss_text_ce: 0.10999874025583267 (0.17991273817420006) + | > loss_mel_ce: 4.424753665924072 (4.429860466003418) + | > loss: 1.1336880922317505 (1.1524433000087733) + | > current_lr: 5e-06 + | > step_time: 0.3863 (0.24503732204437254) + | > loader_time: 0.0281 (0.01575596475601196) + + + --> TIME: 2024-10-19 22:51:34 -- STEP: 550/69059 -- GLOBAL_STEP: 550 + | > loss_text_ce: 0.11246706545352936 (0.17428078958934007) + | > loss_mel_ce: 3.878818988800049 (4.4123552088304) + | > loss: 0.9978215098381042 (1.1466589990529141) + | > current_lr: 5e-06 + | > step_time: 0.3646 (0.2449898143248124) + | > loader_time: 0.0187 (0.015511762879111544) + + + --> TIME: 2024-10-19 22:51:53 -- STEP: 600/69059 -- GLOBAL_STEP: 600 + | > loss_text_ce: 0.10562771558761597 (0.16918049518018966) + | > loss_mel_ce: 4.376893997192383 (4.394590772390364) + | > loss: 1.1206303834915161 (1.140942816535631) + | > current_lr: 5e-06 + | > step_time: 0.158 (0.2445823756853739) + | > loader_time: 0.0109 (0.015306750535964959) + + + --> TIME: 2024-10-19 22:52:10 -- STEP: 650/69059 -- GLOBAL_STEP: 650 + | > loss_text_ce: 0.11017245799303055 (0.16470966486976707) + | > loss_mel_ce: 4.036706447601318 (4.377205421374391) + | > loss: 1.0367196798324585 (1.13547877054948) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.24359152427086458) + | > loader_time: 0.0107 (0.015109201944791349) + + + --> TIME: 2024-10-19 22:52:28 -- STEP: 700/69059 -- GLOBAL_STEP: 700 + | > loss_text_ce: 0.1017698273062706 (0.16058623723685744) + | > loss_mel_ce: 3.99800443649292 (4.359631478105275) + | > loss: 1.0249435901641846 (1.130054428662572) + | > current_lr: 5e-06 + | > step_time: 0.1638 (0.24307773998805451) + | > loader_time: 0.0108 (0.014914841311318528) + + + --> TIME: 2024-10-19 22:52:47 -- STEP: 750/69059 -- GLOBAL_STEP: 750 + | > loss_text_ce: 0.10423920303583145 (0.15676658228039744) + | > loss_mel_ce: 3.860238790512085 (4.344730758984885) + | > loss: 0.9911195039749146 (1.1253743349711092) + | > current_lr: 5e-06 + | > step_time: 0.1787 (0.24371256192525223) + | > loader_time: 0.011 (0.014804238319396968) + + + --> TIME: 2024-10-19 22:53:05 -- STEP: 800/69059 -- GLOBAL_STEP: 800 + | > loss_text_ce: 0.09933288395404816 (0.1533701742626728) + | > loss_mel_ce: 4.005183696746826 (4.33089317202568) + | > loss: 1.026129126548767 (1.1210658358037464) + | > current_lr: 5e-06 + | > step_time: 0.2075 (0.2430923873186111) + | > loader_time: 0.0123 (0.014645610451698298) + + + --> TIME: 2024-10-19 22:53:25 -- STEP: 850/69059 -- GLOBAL_STEP: 850 + | > loss_text_ce: 0.10210375487804413 (0.1501250039479313) + | > loss_mel_ce: 4.246623992919922 (4.318951423308427) + | > loss: 1.0871819257736206 (1.1172691063319924) + | > current_lr: 5e-06 + | > step_time: 0.2567 (0.24462575491736913) + | > loader_time: 0.0116 (0.014568162525401392) + + + --> TIME: 2024-10-19 22:53:44 -- STEP: 900/69059 -- GLOBAL_STEP: 900 + | > loss_text_ce: 0.10321639478206635 (0.14710954214135818) + | > loss_mel_ce: 3.965803623199463 (4.303668042553791) + | > loss: 1.0172549486160278 (1.1126943954494253) + | > current_lr: 5e-06 + | > step_time: 0.153 (0.2448899051878187) + | > loader_time: 0.0111 (0.01445620563295152) + + + --> TIME: 2024-10-19 22:54:03 -- STEP: 950/69059 -- GLOBAL_STEP: 950 + | > loss_text_ce: 0.08765687048435211 (0.14430824020975527) + | > loss_mel_ce: 4.2044148445129395 (4.294632383396747) + | > loss: 1.073017954826355 (1.109735155168332) + | > current_lr: 5e-06 + | > step_time: 0.178 (0.24550390619980658) + | > loader_time: 0.0106 (0.014324996094954637) + + + --> TIME: 2024-10-19 22:54:21 -- STEP: 1000/69059 -- GLOBAL_STEP: 1000 + | > loss_text_ce: 0.09367603808641434 (0.1416385157033802) + | > loss_mel_ce: 4.033923149108887 (4.282719522714611) + | > loss: 1.0318998098373413 (1.1060895090103144) + | > current_lr: 5e-06 + | > step_time: 0.1631 (0.24501673626899717) + | > loader_time: 0.0108 (0.014211693286895748) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_1000.pth + + --> TIME: 2024-10-19 22:54:52 -- STEP: 1050/69059 -- GLOBAL_STEP: 1050 + | > loss_text_ce: 0.09259561449289322 (0.13919052941458576) + | > loss_mel_ce: 4.171869277954102 (4.271708644004092) + | > loss: 1.066116213798523 (1.102724792843773) + | > current_lr: 5e-06 + | > step_time: 0.1942 (0.2444310930797032) + | > loader_time: 0.0107 (0.014118079685029528) + + + --> TIME: 2024-10-19 22:55:12 -- STEP: 1100/69059 -- GLOBAL_STEP: 1100 + | > loss_text_ce: 0.08788391202688217 (0.13681090500544427) + | > loss_mel_ce: 4.112125873565674 (4.262123870416118) + | > loss: 1.0500024557113647 (1.0997336933829565) + | > current_lr: 5e-06 + | > step_time: 0.1875 (0.2451513240554116) + | > loader_time: 0.0109 (0.01407257556915283) + + + --> TIME: 2024-10-19 22:55:30 -- STEP: 1150/69059 -- GLOBAL_STEP: 1150 + | > loss_text_ce: 0.0875265896320343 (0.13461113863665136) + | > loss_mel_ce: 3.5117266178131104 (4.251089920790295) + | > loss: 0.8998132944107056 (1.096425264451814) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.24517328428185503) + | > loader_time: 0.0104 (0.013997606609178625) + + + --> TIME: 2024-10-19 22:55:51 -- STEP: 1200/69059 -- GLOBAL_STEP: 1200 + | > loss_text_ce: 0.07937305420637131 (0.13250264482572685) + | > loss_mel_ce: 4.063185691833496 (4.242377984921133) + | > loss: 1.0356396436691284 (1.0937201568980996) + | > current_lr: 5e-06 + | > step_time: 0.3141 (0.24655875742435454) + | > loader_time: 0.0125 (0.013971607287724813) + + + --> TIME: 2024-10-19 22:56:09 -- STEP: 1250/69059 -- GLOBAL_STEP: 1250 + | > loss_text_ce: 0.08089389652013779 (0.13044969629049308) + | > loss_mel_ce: 4.25839900970459 (4.234863531684872) + | > loss: 1.0848232507705688 (1.0913283064365382) + | > current_lr: 5e-06 + | > step_time: 0.216 (0.246107488822937) + | > loader_time: 0.0113 (0.013902162933349609) + + + --> TIME: 2024-10-19 22:56:28 -- STEP: 1300/69059 -- GLOBAL_STEP: 1300 + | > loss_text_ce: 0.08105237036943436 (0.12852182351625888) + | > loss_mel_ce: 3.9585678577423096 (4.225687583960015) + | > loss: 1.0099050998687744 (1.088552351502271) + | > current_lr: 5e-06 + | > step_time: 0.1639 (0.24593085362361028) + | > loader_time: 0.0106 (0.01384149459692148) + + + --> TIME: 2024-10-19 22:56:45 -- STEP: 1350/69059 -- GLOBAL_STEP: 1350 + | > loss_text_ce: 0.08034934848546982 (0.12666637616576965) + | > loss_mel_ce: 4.14084529876709 (4.215847199404678) + | > loss: 1.0552986860275269 (1.0856283933586544) + | > current_lr: 5e-06 + | > step_time: 0.1725 (0.24516825252109103) + | > loader_time: 0.0106 (0.013764354387919108) + + + --> TIME: 2024-10-19 22:57:03 -- STEP: 1400/69059 -- GLOBAL_STEP: 1400 + | > loss_text_ce: 0.06949134171009064 (0.12481226253190207) + | > loss_mel_ce: 3.990114212036133 (4.2090087357589105) + | > loss: 1.0149013996124268 (1.0834552491562714) + | > current_lr: 5e-06 + | > step_time: 0.1586 (0.24470067756516592) + | > loader_time: 0.0106 (0.013705356631960187) + + + --> TIME: 2024-10-19 22:57:23 -- STEP: 1450/69059 -- GLOBAL_STEP: 1450 + | > loss_text_ce: 0.0696084201335907 (0.12305436343982298) + | > loss_mel_ce: 3.8291523456573486 (4.202211525193571) + | > loss: 0.9746901988983154 (1.0813164717164552) + | > current_lr: 5e-06 + | > step_time: 0.1857 (0.24566670384900324) + | > loader_time: 0.0108 (0.013691413813623888) + + + --> TIME: 2024-10-19 22:57:41 -- STEP: 1500/69059 -- GLOBAL_STEP: 1500 + | > loss_text_ce: 0.06782721728086472 (0.121370613053441) + | > loss_mel_ce: 4.101085662841797 (4.194682565530137) + | > loss: 1.0422282218933105 (1.0790132942597086) + | > current_lr: 5e-06 + | > step_time: 0.1651 (0.24535520442326864) + | > loader_time: 0.0112 (0.013640331109364828) + + + --> TIME: 2024-10-19 22:58:00 -- STEP: 1550/69059 -- GLOBAL_STEP: 1550 + | > loss_text_ce: 0.06400910019874573 (0.11972691418182461) + | > loss_mel_ce: 3.7356314659118652 (4.188881323414464) + | > loss: 0.9499101638793945 (1.0771520592320358) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24550191325526083) + | > loader_time: 0.0105 (0.013585635462114888) + + + --> TIME: 2024-10-19 22:58:18 -- STEP: 1600/69059 -- GLOBAL_STEP: 1600 + | > loss_text_ce: 0.06751026213169098 (0.11814766575815153) + | > loss_mel_ce: 3.6840991973876953 (4.183415286242953) + | > loss: 0.9379023909568787 (1.0753907380253083) + | > current_lr: 5e-06 + | > step_time: 0.1578 (0.2450675457715988) + | > loader_time: 0.0107 (0.013554479777812958) + + + --> TIME: 2024-10-19 22:58:36 -- STEP: 1650/69059 -- GLOBAL_STEP: 1650 + | > loss_text_ce: 0.06276921927928925 (0.11663285614866195) + | > loss_mel_ce: 3.9842259883880615 (4.177782296411917) + | > loss: 1.0117487907409668 (1.0736037880724132) + | > current_lr: 5e-06 + | > step_time: 0.1846 (0.24482056559938373) + | > loader_time: 0.0105 (0.013504732016361121) + + + --> TIME: 2024-10-19 22:58:55 -- STEP: 1700/69059 -- GLOBAL_STEP: 1700 + | > loss_text_ce: 0.06465183943510056 (0.11511499194537889) + | > loss_mel_ce: 4.163299560546875 (4.172456760686984) + | > loss: 1.0569878816604614 (1.071892938087968) + | > current_lr: 5e-06 + | > step_time: 0.3091 (0.24495698928833007) + | > loader_time: 0.0123 (0.013467357859892004) + + + --> TIME: 2024-10-19 22:59:13 -- STEP: 1750/69059 -- GLOBAL_STEP: 1750 + | > loss_text_ce: 0.06157480180263519 (0.11365701220503872) + | > loss_mel_ce: 3.8929028511047363 (4.166287985801692) + | > loss: 0.9886193871498108 (1.069986249412809) + | > current_lr: 5e-06 + | > step_time: 0.2061 (0.24475779778616769) + | > loader_time: 0.0115 (0.013421500478472029) + + + --> TIME: 2024-10-19 22:59:32 -- STEP: 1800/69059 -- GLOBAL_STEP: 1800 + | > loss_text_ce: 0.06856907904148102 (0.11223371196124286) + | > loss_mel_ce: 3.8208396434783936 (4.161387041409806) + | > loss: 0.9723522067070007 (1.068405188288954) + | > current_lr: 5e-06 + | > step_time: 0.1581 (0.24496189951896669) + | > loader_time: 0.0111 (0.01339272830221388) + + + --> TIME: 2024-10-19 22:59:50 -- STEP: 1850/69059 -- GLOBAL_STEP: 1850 + | > loss_text_ce: 0.060330044478178024 (0.11088404067666141) + | > loss_mel_ce: 3.749816417694092 (4.1548570152231115) + | > loss: 0.9525366425514221 (1.066435264123453) + | > current_lr: 5e-06 + | > step_time: 0.194 (0.2448008056589075) + | > loader_time: 0.0107 (0.013362893284978094) + + + --> TIME: 2024-10-19 23:00:09 -- STEP: 1900/69059 -- GLOBAL_STEP: 1900 + | > loss_text_ce: 0.05843694135546684 (0.10957603519880454) + | > loss_mel_ce: 3.8580379486083984 (4.149307305185414) + | > loss: 0.9791187047958374 (1.0647208351837967) + | > current_lr: 5e-06 + | > step_time: 0.1656 (0.24487752613268401) + | > loader_time: 0.0109 (0.013348042086551064) + + + --> TIME: 2024-10-19 23:00:27 -- STEP: 1950/69059 -- GLOBAL_STEP: 1950 + | > loss_text_ce: 0.06305411458015442 (0.1083149295319349) + | > loss_mel_ce: 3.621310234069824 (4.144303549130754) + | > loss: 0.9210910797119141 (1.0631546196876438) + | > current_lr: 5e-06 + | > step_time: 0.2615 (0.24480778302901832) + | > loader_time: 0.0116 (0.013315111918327136) + + + --> TIME: 2024-10-19 23:00:45 -- STEP: 2000/69059 -- GLOBAL_STEP: 2000 + | > loss_text_ce: 0.05601263418793678 (0.10709203919582067) + | > loss_mel_ce: 3.556987762451172 (4.139470639705654) + | > loss: 0.9032500982284546 (1.0616406699121008) + | > current_lr: 5e-06 + | > step_time: 0.1576 (0.24458309996128083) + | > loader_time: 0.0106 (0.013282333493232728) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_2000.pth + + --> TIME: 2024-10-19 23:01:23 -- STEP: 2050/69059 -- GLOBAL_STEP: 2050 + | > loss_text_ce: 0.06410832703113556 (0.10588896331627191) + | > loss_mel_ce: 4.000243663787842 (4.13435949383712) + | > loss: 1.0160880088806152 (1.0600621142329247) + | > current_lr: 5e-06 + | > step_time: 0.4444 (0.24431447040743945) + | > loader_time: 0.0136 (0.013252315288636742) + + + --> TIME: 2024-10-19 23:01:42 -- STEP: 2100/69059 -- GLOBAL_STEP: 2100 + | > loss_text_ce: 0.06505782902240753 (0.10475704486348797) + | > loss_mel_ce: 3.75443959236145 (4.129762740248722) + | > loss: 0.9548743367195129 (1.0586299462261664) + | > current_lr: 5e-06 + | > step_time: 0.221 (0.2444660292352949) + | > loader_time: 0.0107 (0.013231485003516787) + + + --> TIME: 2024-10-19 23:02:02 -- STEP: 2150/69059 -- GLOBAL_STEP: 2150 + | > loss_text_ce: 0.0588456392288208 (0.10368685242741603) + | > loss_mel_ce: 3.6094536781311035 (4.125783104785651) + | > loss: 0.9170747995376587 (1.0573674891438603) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.2448491818405861) + | > loader_time: 0.0107 (0.01321672561556794) + + + --> TIME: 2024-10-19 23:02:21 -- STEP: 2200/69059 -- GLOBAL_STEP: 2200 + | > loss_text_ce: 0.062258265912532806 (0.10267997404899104) + | > loss_mel_ce: 4.0013837814331055 (4.120954286943782) + | > loss: 1.0159105062484741 (1.0559085651419389) + | > current_lr: 5e-06 + | > step_time: 0.3238 (0.2449620660868558) + | > loader_time: 0.0123 (0.013187857432798896) + + + --> TIME: 2024-10-19 23:02:39 -- STEP: 2250/69059 -- GLOBAL_STEP: 2250 + | > loss_text_ce: 0.0607147291302681 (0.10166863545278702) + | > loss_mel_ce: 4.2442426681518555 (4.11629843319787) + | > loss: 1.0762393474578857 (1.0544917671415552) + | > current_lr: 5e-06 + | > step_time: 0.2438 (0.2447140187157525) + | > loader_time: 0.0115 (0.013156991216871462) + + + --> TIME: 2024-10-19 23:02:57 -- STEP: 2300/69059 -- GLOBAL_STEP: 2300 + | > loss_text_ce: 0.05426686629652977 (0.10071106363087882) + | > loss_mel_ce: 3.977905750274658 (4.111614592489988) + | > loss: 1.0080431699752808 (1.0530814139998486) + | > current_lr: 5e-06 + | > step_time: 0.1635 (0.24463536241780157) + | > loader_time: 0.0107 (0.01313344831052032) + + + --> TIME: 2024-10-19 23:03:16 -- STEP: 2350/69059 -- GLOBAL_STEP: 2350 + | > loss_text_ce: 0.05568445101380348 (0.09979596726437819) + | > loss_mel_ce: 4.079419136047363 (4.107172711554993) + | > loss: 1.0337759256362915 (1.0517421696287528) + | > current_lr: 5e-06 + | > step_time: 0.4189 (0.24469409546953566) + | > loader_time: 0.0129 (0.013110532456255961) + + + --> TIME: 2024-10-19 23:03:33 -- STEP: 2400/69059 -- GLOBAL_STEP: 2400 + | > loss_text_ce: 0.050804924219846725 (0.09889819633836533) + | > loss_mel_ce: 3.719665765762329 (4.10211707095305) + | > loss: 0.942617654800415 (1.050253816743693) + | > current_lr: 5e-06 + | > step_time: 0.1833 (0.24416771670182547) + | > loader_time: 0.0114 (0.013079029222329442) + + + --> TIME: 2024-10-19 23:03:52 -- STEP: 2450/69059 -- GLOBAL_STEP: 2450 + | > loss_text_ce: 0.059565141797065735 (0.0980295563473992) + | > loss_mel_ce: 3.853571653366089 (4.0975918043876165) + | > loss: 0.9782841801643372 (1.0489053400195383) + | > current_lr: 5e-06 + | > step_time: 0.2786 (0.24423073797809836) + | > loader_time: 0.012 (0.013054829908876986) + + + --> TIME: 2024-10-19 23:04:10 -- STEP: 2500/69059 -- GLOBAL_STEP: 2500 + | > loss_text_ce: 0.05738230049610138 (0.09718903902769081) + | > loss_mel_ce: 3.7446444034576416 (4.093490422534943) + | > loss: 0.9505066871643066 (1.0476698652267464) + | > current_lr: 5e-06 + | > step_time: 0.2485 (0.24415877380371093) + | > loader_time: 0.0114 (0.013029421710968) + + + --> TIME: 2024-10-19 23:04:29 -- STEP: 2550/69059 -- GLOBAL_STEP: 2550 + | > loss_text_ce: 0.05330777168273926 (0.0963746939380378) + | > loss_mel_ce: 4.240808486938477 (4.089953881899517) + | > loss: 1.0735290050506592 (1.0465821437040979) + | > current_lr: 5e-06 + | > step_time: 0.2951 (0.24447077975553624) + | > loader_time: 0.0118 (0.013008702970018557) + + + --> TIME: 2024-10-19 23:04:49 -- STEP: 2600/69059 -- GLOBAL_STEP: 2600 + | > loss_text_ce: 0.05627383291721344 (0.0956042603704218) + | > loss_mel_ce: 3.9000959396362305 (4.086011801958086) + | > loss: 0.9890924692153931 (1.0454040152522244) + | > current_lr: 5e-06 + | > step_time: 0.1644 (0.24479702087549063) + | > loader_time: 0.0113 (0.013004047687237064) + + + --> TIME: 2024-10-19 23:05:07 -- STEP: 2650/69059 -- GLOBAL_STEP: 2650 + | > loss_text_ce: 0.059830036014318466 (0.09486349638902906) + | > loss_mel_ce: 3.8829853534698486 (4.0820781279509974) + | > loss: 0.9857038259506226 (1.0442354058094752) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.24474286592231606) + | > loader_time: 0.0104 (0.012982751468442504) + + + --> TIME: 2024-10-19 23:05:27 -- STEP: 2700/69059 -- GLOBAL_STEP: 2700 + | > loss_text_ce: 0.057055190205574036 (0.09413117132529057) + | > loss_mel_ce: 3.914095640182495 (4.078434617254474) + | > loss: 0.9927877187728882 (1.0431414468420879) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.2449570646992436) + | > loader_time: 0.0107 (0.012967486646440278) + + + --> TIME: 2024-10-19 23:05:45 -- STEP: 2750/69059 -- GLOBAL_STEP: 2750 + | > loss_text_ce: 0.05911615118384361 (0.09341719400476324) + | > loss_mel_ce: 4.031272888183594 (4.074315213983713) + | > loss: 1.022597312927246 (1.0419331016757274) + | > current_lr: 5e-06 + | > step_time: 0.3947 (0.24485181522369384) + | > loader_time: 0.0161 (0.01294834726507012) + + + --> TIME: 2024-10-19 23:06:04 -- STEP: 2800/69059 -- GLOBAL_STEP: 2800 + | > loss_text_ce: 0.05606602877378464 (0.09273461623915602) + | > loss_mel_ce: 3.8986992835998535 (4.070694002338826) + | > loss: 0.9886913299560547 (1.0408571543438088) + | > current_lr: 5e-06 + | > step_time: 0.3337 (0.24501532069274357) + | > loader_time: 0.0128 (0.012929019417081545) + + + --> TIME: 2024-10-19 23:06:21 -- STEP: 2850/69059 -- GLOBAL_STEP: 2850 + | > loss_text_ce: 0.052219562232494354 (0.09206971768747295) + | > loss_mel_ce: 3.5624022483825684 (4.066505895832134) + | > loss: 0.903655469417572 (1.0396439030295928) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.2447062503245839) + | > loader_time: 0.0108 (0.012909067890100297) + + + --> TIME: 2024-10-19 23:06:41 -- STEP: 2900/69059 -- GLOBAL_STEP: 2900 + | > loss_text_ce: 0.05405086278915405 (0.0914257599387703) + | > loss_mel_ce: 4.232316493988037 (4.06327337117031) + | > loss: 1.071591854095459 (1.0386747823295934) + | > current_lr: 5e-06 + | > step_time: 0.3391 (0.24516136202318914) + | > loader_time: 0.0124 (0.01289901470315866) + + + --> TIME: 2024-10-19 23:07:00 -- STEP: 2950/69059 -- GLOBAL_STEP: 2950 + | > loss_text_ce: 0.06172400712966919 (0.09079265477293627) + | > loss_mel_ce: 3.9180195331573486 (4.059609037092181) + | > loss: 0.9949358701705933 (1.0376004224914634) + | > current_lr: 5e-06 + | > step_time: 0.3579 (0.24516837887844797) + | > loader_time: 0.0132 (0.012882576554508519) + + + --> TIME: 2024-10-19 23:07:18 -- STEP: 3000/69059 -- GLOBAL_STEP: 3000 + | > loss_text_ce: 0.05317004397511482 (0.09017035951962071) + | > loss_mel_ce: 3.8724138736724854 (4.054939047733947) + | > loss: 0.981395959854126 (1.0362773513595285) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.24506164145469667) + | > loader_time: 0.0106 (0.012865348895390814) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_3000.pth + + --> TIME: 2024-10-19 23:07:56 -- STEP: 3050/69059 -- GLOBAL_STEP: 3050 + | > loss_text_ce: 0.058816298842430115 (0.08958689314908667) + | > loss_mel_ce: 4.177435398101807 (4.052077963156781) + | > loss: 1.0590629577636719 (1.0354162135866836) + | > current_lr: 5e-06 + | > step_time: 0.3535 (0.2452566276612829) + | > loader_time: 0.0163 (0.012859432658211116) + + + --> TIME: 2024-10-19 23:08:16 -- STEP: 3100/69059 -- GLOBAL_STEP: 3100 + | > loss_text_ce: 0.05225648730993271 (0.08899778135120867) + | > loss_mel_ce: 3.52755069732666 (4.049394690682811) + | > loss: 0.8949518203735352 (1.0345981174899697) + | > current_lr: 5e-06 + | > step_time: 0.1564 (0.24557567373398811) + | > loader_time: 0.0119 (0.012852721445022077) + + + --> TIME: 2024-10-19 23:08:36 -- STEP: 3150/69059 -- GLOBAL_STEP: 3150 + | > loss_text_ce: 0.05638252943754196 (0.08843211695551871) + | > loss_mel_ce: 3.696380376815796 (4.046571773877219) + | > loss: 0.9381906986236572 (1.03375097218014) + | > current_lr: 5e-06 + | > step_time: 0.1798 (0.24601844704340375) + | > loader_time: 0.0132 (0.012843127402048248) + + + --> TIME: 2024-10-19 23:08:55 -- STEP: 3200/69059 -- GLOBAL_STEP: 3200 + | > loss_text_ce: 0.054836519062519073 (0.08787966135190799) + | > loss_mel_ce: 4.12384557723999 (4.0431763614714145) + | > loss: 1.044670581817627 (1.0327640051580966) + | > current_lr: 5e-06 + | > step_time: 0.4489 (0.24604346916079523) + | > loader_time: 0.0146 (0.012831838130950915) + + + --> TIME: 2024-10-19 23:09:16 -- STEP: 3250/69059 -- GLOBAL_STEP: 3250 + | > loss_text_ce: 0.05327862501144409 (0.08734721159934995) + | > loss_mel_ce: 4.124325752258301 (4.040207889556884) + | > loss: 1.0444010496139526 (1.0318887747434475) + | > current_lr: 5e-06 + | > step_time: 0.3741 (0.24641809800954964) + | > loader_time: 0.0128 (0.012823832511901842) + + + --> TIME: 2024-10-19 23:09:34 -- STEP: 3300/69059 -- GLOBAL_STEP: 3300 + | > loss_text_ce: 0.0556054525077343 (0.08681979405270379) + | > loss_mel_ce: 3.829547166824341 (4.037217605908711) + | > loss: 0.9712881445884705 (1.031009349479821) + | > current_lr: 5e-06 + | > step_time: 0.1973 (0.24633563301780007) + | > loader_time: 0.0114 (0.012812176906701278) + + + --> TIME: 2024-10-19 23:09:52 -- STEP: 3350/69059 -- GLOBAL_STEP: 3350 + | > loss_text_ce: 0.05443510785698891 (0.08631290650745824) + | > loss_mel_ce: 3.6743879318237305 (4.034318862744227) + | > loss: 0.9322057366371155 (1.0301579418467066) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.24624529368841827) + | > loader_time: 0.0107 (0.012800860760816874) + + + --> TIME: 2024-10-19 23:10:09 -- STEP: 3400/69059 -- GLOBAL_STEP: 3400 + | > loss_text_ce: 0.05562814697623253 (0.08580609962344168) + | > loss_mel_ce: 3.7173900604248047 (4.0313874617744805) + | > loss: 0.9432545304298401 (1.0292983898520485) + | > current_lr: 5e-06 + | > step_time: 0.1485 (0.24587884089526008) + | > loader_time: 0.0103 (0.012779442282283995) + + + --> TIME: 2024-10-19 23:10:28 -- STEP: 3450/69059 -- GLOBAL_STEP: 3450 + | > loss_text_ce: 0.04778096452355385 (0.08531993946195511) + | > loss_mel_ce: 3.874321937561035 (4.02826262404952) + | > loss: 0.9805257320404053 (1.0283956404354269) + | > current_lr: 5e-06 + | > step_time: 0.2151 (0.24591006513954936) + | > loader_time: 0.011 (0.012767354232677502) + + + --> TIME: 2024-10-19 23:10:45 -- STEP: 3500/69059 -- GLOBAL_STEP: 3500 + | > loss_text_ce: 0.05177321657538414 (0.08483412744637045) + | > loss_mel_ce: 3.748081922531128 (4.0249305238723645) + | > loss: 0.9499638080596924 (1.027441162381855) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.24563034697941372) + | > loader_time: 0.0105 (0.012747989995138974) + + + --> TIME: 2024-10-19 23:11:03 -- STEP: 3550/69059 -- GLOBAL_STEP: 3550 + | > loss_text_ce: 0.057300347834825516 (0.08436951584589311) + | > loss_mel_ce: 3.5488407611846924 (4.021654228962629) + | > loss: 0.9015352725982666 (1.026505935762971) + | > current_lr: 5e-06 + | > step_time: 0.2151 (0.24541296945491306) + | > loader_time: 0.0113 (0.012733226426890187) + + + --> TIME: 2024-10-19 23:11:20 -- STEP: 3600/69059 -- GLOBAL_STEP: 3600 + | > loss_text_ce: 0.04895898699760437 (0.08391605445183813) + | > loss_mel_ce: 3.6627368927001953 (4.019016283949202) + | > loss: 0.9279239773750305 (1.0257330841488321) + | > current_lr: 5e-06 + | > step_time: 0.1734 (0.24501842538515728) + | > loader_time: 0.0115 (0.012712670432196711) + + + --> TIME: 2024-10-19 23:11:39 -- STEP: 3650/69059 -- GLOBAL_STEP: 3650 + | > loss_text_ce: 0.04623499885201454 (0.08348102260432014) + | > loss_mel_ce: 3.684278726577759 (4.016074813816632) + | > loss: 0.9326284527778625 (1.02488895875134) + | > current_lr: 5e-06 + | > step_time: 0.1843 (0.24529166358790985) + | > loader_time: 0.0109 (0.012703959843883764) + + + --> TIME: 2024-10-19 23:11:57 -- STEP: 3700/69059 -- GLOBAL_STEP: 3700 + | > loss_text_ce: 0.054940178990364075 (0.08305093761734864) + | > loss_mel_ce: 3.6010477542877197 (4.013568388062541) + | > loss: 0.9139969944953918 (1.0241548311388187) + | > current_lr: 5e-06 + | > step_time: 0.3166 (0.2451945323557467) + | > loader_time: 0.0124 (0.01268881185634715) + + + --> TIME: 2024-10-19 23:12:17 -- STEP: 3750/69059 -- GLOBAL_STEP: 3750 + | > loss_text_ce: 0.05661119520664215 (0.08263179731269676) + | > loss_mel_ce: 3.9613640308380127 (4.011045390129074) + | > loss: 1.0044938325881958 (1.0234192965189626) + | > current_lr: 5e-06 + | > step_time: 0.2639 (0.2454115960439046) + | > loader_time: 0.0135 (0.012677513376871733) + + + --> TIME: 2024-10-19 23:12:35 -- STEP: 3800/69059 -- GLOBAL_STEP: 3800 + | > loss_text_ce: 0.049454983323812485 (0.08221463949762675) + | > loss_mel_ce: 3.8226568698883057 (4.007853990416762) + | > loss: 0.9680279493331909 (1.022517157149944) + | > current_lr: 5e-06 + | > step_time: 0.296 (0.24525841298856232) + | > loader_time: 0.0121 (0.0126629288573014) + + + --> TIME: 2024-10-19 23:12:54 -- STEP: 3850/69059 -- GLOBAL_STEP: 3850 + | > loss_text_ce: 0.04678835719823837 (0.08180926125254723) + | > loss_mel_ce: 4.005477428436279 (4.00512358108123) + | > loss: 1.0130664110183716 (1.0217332103345325) + | > current_lr: 5e-06 + | > step_time: 0.2112 (0.24536061224999367) + | > loader_time: 0.0111 (0.012652693042507409) + + + --> TIME: 2024-10-19 23:13:13 -- STEP: 3900/69059 -- GLOBAL_STEP: 3900 + | > loss_text_ce: 0.04983033984899521 (0.08140554877427907) + | > loss_mel_ce: 3.748762607574463 (4.00208383602972) + | > loss: 0.9496482610702515 (1.0208723459946822) + | > current_lr: 5e-06 + | > step_time: 0.1523 (0.24540421577600333) + | > loader_time: 0.0101 (0.012640813925327388) + + + --> TIME: 2024-10-19 23:13:31 -- STEP: 3950/69059 -- GLOBAL_STEP: 3950 + | > loss_text_ce: 0.04662459343671799 (0.08102970183555837) + | > loss_mel_ce: 3.747682809829712 (3.999170341370963) + | > loss: 0.9485768675804138 (1.0200500105906152) + | > current_lr: 5e-06 + | > step_time: 0.2015 (0.24536445412454727) + | > loader_time: 0.0109 (0.012626375669165493) + + + --> TIME: 2024-10-19 23:13:50 -- STEP: 4000/69059 -- GLOBAL_STEP: 4000 + | > loss_text_ce: 0.048626720905303955 (0.08064724720735102) + | > loss_mel_ce: 3.6373660564422607 (3.996222546875459) + | > loss: 0.92149817943573 (1.0192174482494607) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.24538100701570512) + | > loader_time: 0.011 (0.0126127209663391) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_4000.pth + + --> TIME: 2024-10-19 23:14:23 -- STEP: 4050/69059 -- GLOBAL_STEP: 4050 + | > loss_text_ce: 0.05422494187951088 (0.08027469530333706) + | > loss_mel_ce: 3.7344884872436523 (3.9933313887207422) + | > loss: 0.9471783638000488 (1.0184015207231794) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.24526031170362309) + | > loader_time: 0.0108 (0.012605073540299017) + + + --> TIME: 2024-10-19 23:14:41 -- STEP: 4100/69059 -- GLOBAL_STEP: 4100 + | > loss_text_ce: 0.048128996044397354 (0.0799101300037852) + | > loss_mel_ce: 3.813293695449829 (3.990760178391509) + | > loss: 0.9653556942939758 (1.017667576830563) + | > current_lr: 5e-06 + | > step_time: 0.1557 (0.2452218900075773) + | > loader_time: 0.0103 (0.012595617887450417) + + + --> TIME: 2024-10-19 23:15:01 -- STEP: 4150/69059 -- GLOBAL_STEP: 4150 + | > loss_text_ce: 0.052940454334020615 (0.07956504111189439) + | > loss_mel_ce: 3.8037831783294678 (3.9887170125202784) + | > loss: 0.9641808867454529 (1.0170705131306714) + | > current_lr: 5e-06 + | > step_time: 0.2844 (0.2455353530631008) + | > loader_time: 0.0112 (0.012595301019140026) + + + --> TIME: 2024-10-19 23:15:19 -- STEP: 4200/69059 -- GLOBAL_STEP: 4200 + | > loss_text_ce: 0.04771630838513374 (0.07921872089927395) + | > loss_mel_ce: 3.5244393348693848 (3.986028704927064) + | > loss: 0.8930389285087585 (1.0163118561108913) + | > current_lr: 5e-06 + | > step_time: 0.1514 (0.24536524273100355) + | > loader_time: 0.0105 (0.012580224048523667) + + + --> TIME: 2024-10-19 23:15:35 -- STEP: 4250/69059 -- GLOBAL_STEP: 4250 + | > loss_text_ce: 0.053990963846445084 (0.07886897211740998) + | > loss_mel_ce: 3.630352735519409 (3.983283456297465) + | > loss: 0.9210859537124634 (1.0155381067921145) + | > current_lr: 5e-06 + | > step_time: 0.2232 (0.2450396739735323) + | > loader_time: 0.0109 (0.012563091165879182) + + + --> TIME: 2024-10-19 23:15:55 -- STEP: 4300/69059 -- GLOBAL_STEP: 4300 + | > loss_text_ce: 0.04915660619735718 (0.07852911985371004) + | > loss_mel_ce: 3.865298271179199 (3.981281495371513) + | > loss: 0.9786137342453003 (1.0149526535217168) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.2452020622408667) + | > loader_time: 0.0101 (0.01255599859148956) + + + --> TIME: 2024-10-19 23:16:14 -- STEP: 4350/69059 -- GLOBAL_STEP: 4350 + | > loss_text_ce: 0.046060264110565186 (0.07819221514994389) + | > loss_mel_ce: 3.913447618484497 (3.9789674157109585) + | > loss: 0.9898769855499268 (1.014289907408858) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24523728463841582) + | > loader_time: 0.0114 (0.01254891066715634) + + + --> TIME: 2024-10-19 23:16:31 -- STEP: 4400/69059 -- GLOBAL_STEP: 4400 + | > loss_text_ce: 0.045438770204782486 (0.0778688956856389) + | > loss_mel_ce: 3.6974942684173584 (3.9759406764940617) + | > loss: 0.9357332587242126 (1.0134523927081724) + | > current_lr: 5e-06 + | > step_time: 0.1657 (0.24505971534685655) + | > loader_time: 0.0109 (0.012538315057754507) + + + --> TIME: 2024-10-19 23:16:50 -- STEP: 4450/69059 -- GLOBAL_STEP: 4450 + | > loss_text_ce: 0.051282528787851334 (0.07755493787232409) + | > loss_mel_ce: 3.693950891494751 (3.9732498542378294) + | > loss: 0.9363083839416504 (1.0127011977152889) + | > current_lr: 5e-06 + | > step_time: 0.1773 (0.24513017306167087) + | > loader_time: 0.0114 (0.012534256356485765) + + + --> TIME: 2024-10-19 23:17:08 -- STEP: 4500/69059 -- GLOBAL_STEP: 4500 + | > loss_text_ce: 0.04411055147647858 (0.07724089605857928) + | > loss_mel_ce: 3.5011303424835205 (3.9706520829730403) + | > loss: 0.8863102197647095 (1.0119732444551266) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.24502240159776476) + | > loader_time: 0.0106 (0.012528504265679245) + + + --> TIME: 2024-10-19 23:17:26 -- STEP: 4550/69059 -- GLOBAL_STEP: 4550 + | > loss_text_ce: 0.04941542074084282 (0.07693133924852362) + | > loss_mel_ce: 3.8679401874542236 (3.968358433010798) + | > loss: 0.9793388843536377 (1.0113224427909648) + | > current_lr: 5e-06 + | > step_time: 0.2782 (0.2448965459865528) + | > loader_time: 0.0119 (0.012516055107116692) + + + --> TIME: 2024-10-19 23:17:44 -- STEP: 4600/69059 -- GLOBAL_STEP: 4600 + | > loss_text_ce: 0.04733306169509888 (0.07663008404976647) + | > loss_mel_ce: 3.808915376663208 (3.965809703650666) + | > loss: 0.9640620946884155 (1.0106099466396419) + | > current_lr: 5e-06 + | > step_time: 0.1526 (0.24490179155183875) + | > loader_time: 0.011 (0.012506995252940961) + + + --> TIME: 2024-10-19 23:18:02 -- STEP: 4650/69059 -- GLOBAL_STEP: 4650 + | > loss_text_ce: 0.044091179966926575 (0.07632883149129088) + | > loss_mel_ce: 3.445472002029419 (3.9629976632518) + | > loss: 0.8723908066749573 (1.0098316233901572) + | > current_lr: 5e-06 + | > step_time: 0.1947 (0.24478614776365218) + | > loader_time: 0.0112 (0.012502547387153862) + + + --> TIME: 2024-10-19 23:18:20 -- STEP: 4700/69059 -- GLOBAL_STEP: 4700 + | > loss_text_ce: 0.05079268664121628 (0.07603695470998263) + | > loss_mel_ce: 3.621063709259033 (3.9602656131094918) + | > loss: 0.9179641008377075 (1.0090756416574442) + | > current_lr: 5e-06 + | > step_time: 0.1565 (0.24470664262771608) + | > loader_time: 0.0113 (0.012497676991401831) + + + --> TIME: 2024-10-19 23:18:38 -- STEP: 4750/69059 -- GLOBAL_STEP: 4750 + | > loss_text_ce: 0.04558158293366432 (0.07574671121333779) + | > loss_mel_ce: 3.520688772201538 (3.9576496275851447) + | > loss: 0.891567587852478 (1.0083490843772898) + | > current_lr: 5e-06 + | > step_time: 0.1711 (0.24463324647200735) + | > loader_time: 0.0109 (0.012490505218505865) + + + --> TIME: 2024-10-19 23:18:57 -- STEP: 4800/69059 -- GLOBAL_STEP: 4800 + | > loss_text_ce: 0.04871422052383423 (0.07546597281082852) + | > loss_mel_ce: 3.7146658897399902 (3.955281311422571) + | > loss: 0.9408450126647949 (1.0076868207628542) + | > current_lr: 5e-06 + | > step_time: 0.1902 (0.24470093746980032) + | > loader_time: 0.011 (0.012483203858137134) + + + --> TIME: 2024-10-19 23:19:16 -- STEP: 4850/69059 -- GLOBAL_STEP: 4850 + | > loss_text_ce: 0.04583296552300453 (0.07519000252497568) + | > loss_mel_ce: 3.747960329055786 (3.9526878791985904) + | > loss: 0.9484483003616333 (1.00696947015438) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24467230187248937) + | > loader_time: 0.011 (0.012475728841172054) + + + --> TIME: 2024-10-19 23:19:34 -- STEP: 4900/69059 -- GLOBAL_STEP: 4900 + | > loss_text_ce: 0.05504792556166649 (0.07491747299323277) + | > loss_mel_ce: 3.8904306888580322 (3.95056337215461) + | > loss: 0.9863696694374084 (1.0063702110246748) + | > current_lr: 5e-06 + | > step_time: 0.359 (0.24468346907168018) + | > loader_time: 0.013 (0.012468014104025713) + + + --> TIME: 2024-10-19 23:19:54 -- STEP: 4950/69059 -- GLOBAL_STEP: 4950 + | > loss_text_ce: 0.04845403879880905 (0.07465249768712308) + | > loss_mel_ce: 4.00543212890625 (3.9483242822415754) + | > loss: 1.0134714841842651 (1.0057441947074863) + | > current_lr: 5e-06 + | > step_time: 0.3582 (0.24486523146581168) + | > loader_time: 0.0115 (0.012463421291775191) + + + --> TIME: 2024-10-19 23:20:12 -- STEP: 5000/69059 -- GLOBAL_STEP: 5000 + | > loss_text_ce: 0.04849144443869591 (0.07439490167424082) + | > loss_mel_ce: 3.9106316566467285 (3.9458948535919043) + | > loss: 0.9897807836532593 (1.0050724385023124) + | > current_lr: 5e-06 + | > step_time: 0.2764 (0.24478563542366027) + | > loader_time: 0.0134 (0.012456556081771864) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_5000.pth + + --> TIME: 2024-10-19 23:20:44 -- STEP: 5050/69059 -- GLOBAL_STEP: 5050 + | > loss_text_ce: 0.04689784720540047 (0.0741386988176273) + | > loss_mel_ce: 3.6225874423980713 (3.943291969299302) + | > loss: 0.9173713326454163 (1.0043576666978338) + | > current_lr: 5e-06 + | > step_time: 0.1883 (0.24480573852463525) + | > loader_time: 0.0137 (0.012451980279223767) + + + --> TIME: 2024-10-19 23:21:01 -- STEP: 5100/69059 -- GLOBAL_STEP: 5100 + | > loss_text_ce: 0.0510975643992424 (0.07387838672408281) + | > loss_mel_ce: 3.8692262172698975 (3.9406977084571095) + | > loss: 0.9800809621810913 (1.0036440235142612) + | > current_lr: 5e-06 + | > step_time: 0.2163 (0.24456956344492295) + | > loader_time: 0.0106 (0.012444182470733057) + + + --> TIME: 2024-10-19 23:21:21 -- STEP: 5150/69059 -- GLOBAL_STEP: 5150 + | > loss_text_ce: 0.050038766115903854 (0.07363230939050316) + | > loss_mel_ce: 3.719588041305542 (3.9384959499581207) + | > loss: 0.9424067139625549 (1.0030320645883237) + | > current_lr: 5e-06 + | > step_time: 0.3047 (0.24471573088932963) + | > loader_time: 0.0122 (0.012441051770182494) + + + --> TIME: 2024-10-19 23:21:38 -- STEP: 5200/69059 -- GLOBAL_STEP: 5200 + | > loss_text_ce: 0.046597208827733994 (0.07338353004234911) + | > loss_mel_ce: 3.6463534832000732 (3.935889605971469) + | > loss: 0.923237681388855 (1.0023182837321194) + | > current_lr: 5e-06 + | > step_time: 0.1636 (0.2444943999327146) + | > loader_time: 0.0104 (0.012432708786084106) + + + --> TIME: 2024-10-19 23:21:57 -- STEP: 5250/69059 -- GLOBAL_STEP: 5250 + | > loss_text_ce: 0.043374061584472656 (0.07313218927028635) + | > loss_mel_ce: 3.6166088581085205 (3.9339604942684945) + | > loss: 0.9149957299232483 (1.0017731706528425) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.244653249331883) + | > loader_time: 0.0131 (0.012429211344037742) + + + --> TIME: 2024-10-19 23:22:18 -- STEP: 5300/69059 -- GLOBAL_STEP: 5300 + | > loss_text_ce: 0.04682283475995064 (0.07289597256987727) + | > loss_mel_ce: 4.072071075439453 (3.931976128839083) + | > loss: 1.0297235250473022 (1.0012180251445395) + | > current_lr: 5e-06 + | > step_time: 0.3359 (0.24491541543096867) + | > loader_time: 0.0134 (0.012428571368163492) + + + --> TIME: 2024-10-19 23:22:36 -- STEP: 5350/69059 -- GLOBAL_STEP: 5350 + | > loss_text_ce: 0.044526711106300354 (0.07266145025249396) + | > loss_mel_ce: 3.5783464908599854 (3.92936010556799) + | > loss: 0.9057183265686035 (1.0005053887968849) + | > current_lr: 5e-06 + | > step_time: 0.1719 (0.2448254676177123) + | > loader_time: 0.0105 (0.012420630900659299) + + + --> TIME: 2024-10-19 23:22:53 -- STEP: 5400/69059 -- GLOBAL_STEP: 5400 + | > loss_text_ce: 0.04930824041366577 (0.07242372302131525) + | > loss_mel_ce: 3.518585443496704 (3.9272555048818805) + | > loss: 0.8919734358787537 (0.9999198068512787) + | > current_lr: 5e-06 + | > step_time: 0.1564 (0.2447391629219055) + | > loader_time: 0.011 (0.012415105545962304) + + + --> TIME: 2024-10-19 23:23:13 -- STEP: 5450/69059 -- GLOBAL_STEP: 5450 + | > loss_text_ce: 0.05117104947566986 (0.07219265267868114) + | > loss_mel_ce: 3.8260045051574707 (3.9254466046761896) + | > loss: 0.9692938923835754 (0.9994098141871438) + | > current_lr: 5e-06 + | > step_time: 0.4209 (0.24484549631766223) + | > loader_time: 0.0136 (0.012416312935155471) + + + --> TIME: 2024-10-19 23:23:30 -- STEP: 5500/69059 -- GLOBAL_STEP: 5500 + | > loss_text_ce: 0.050743211060762405 (0.0719682096866045) + | > loss_mel_ce: 3.9359843730926514 (3.9228532270951573) + | > loss: 0.9966818690299988 (0.9987053590037582) + | > current_lr: 5e-06 + | > step_time: 0.2604 (0.24470924763246016) + | > loader_time: 0.0152 (0.012412087830630221) + + + --> TIME: 2024-10-19 23:23:50 -- STEP: 5550/69059 -- GLOBAL_STEP: 5550 + | > loss_text_ce: 0.05448511987924576 (0.07174493171140424) + | > loss_mel_ce: 3.6402642726898193 (3.920533693932185) + | > loss: 0.9236873388290405 (0.9980696562066785) + | > current_lr: 5e-06 + | > step_time: 0.1845 (0.2448267815993713) + | > loader_time: 0.011 (0.012408450444539393) + + + --> TIME: 2024-10-19 23:24:07 -- STEP: 5600/69059 -- GLOBAL_STEP: 5600 + | > loss_text_ce: 0.04488856717944145 (0.07152166118113597) + | > loss_mel_ce: 3.5506503582000732 (3.917930243611323) + | > loss: 0.8988847136497498 (0.9973629760316417) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24470735290220805) + | > loader_time: 0.0112 (0.012399121522903447) + + + --> TIME: 2024-10-19 23:24:25 -- STEP: 5650/69059 -- GLOBAL_STEP: 5650 + | > loss_text_ce: 0.05069196596741676 (0.07130123989244487) + | > loss_mel_ce: 3.6082687377929688 (3.915590211184666) + | > loss: 0.9147402048110962 (0.9967228626149923) + | > current_lr: 5e-06 + | > step_time: 0.4841 (0.24459584485113092) + | > loader_time: 0.0169 (0.012392815995005385) + + + --> TIME: 2024-10-19 23:24:43 -- STEP: 5700/69059 -- GLOBAL_STEP: 5700 + | > loss_text_ce: 0.05131392180919647 (0.07108578927749606) + | > loss_mel_ce: 3.599846363067627 (3.913525559065622) + | > loss: 0.912790060043335 (0.9961528369104631) + | > current_lr: 5e-06 + | > step_time: 0.3758 (0.2445685356959962) + | > loader_time: 0.0107 (0.012390415710315374) + + + --> TIME: 2024-10-19 23:25:02 -- STEP: 5750/69059 -- GLOBAL_STEP: 5750 + | > loss_text_ce: 0.044475849717855453 (0.07088078296767633) + | > loss_mel_ce: 3.83298659324646 (3.9117662312051396) + | > loss: 0.9693655967712402 (0.9956617533745951) + | > current_lr: 5e-06 + | > step_time: 0.1793 (0.24461796893244203) + | > loader_time: 0.0111 (0.012385005287502127) + + + --> TIME: 2024-10-19 23:25:22 -- STEP: 5800/69059 -- GLOBAL_STEP: 5800 + | > loss_text_ce: 0.048976171761751175 (0.07068291818145023) + | > loss_mel_ce: 3.707655668258667 (3.909912635499021) + | > loss: 0.9391579627990723 (0.9951488883022582) + | > current_lr: 5e-06 + | > step_time: 0.3949 (0.24477394490406432) + | > loader_time: 0.0168 (0.012381941696693162) + + + --> TIME: 2024-10-19 23:25:40 -- STEP: 5850/69059 -- GLOBAL_STEP: 5850 + | > loss_text_ce: 0.05015479773283005 (0.07047739616889732) + | > loss_mel_ce: 3.6084914207458496 (3.90778953116163) + | > loss: 0.9146615266799927 (0.994566731738227) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24471310550331052) + | > loader_time: 0.0102 (0.012375218318058899) + + + --> TIME: 2024-10-19 23:25:57 -- STEP: 5900/69059 -- GLOBAL_STEP: 5900 + | > loss_text_ce: 0.043290961533784866 (0.07027162527065668) + | > loss_mel_ce: 3.348266839981079 (3.905826063479395) + | > loss: 0.8478894233703613 (0.9940244220980121) + | > current_lr: 5e-06 + | > step_time: 0.1579 (0.24449688188100266) + | > loader_time: 0.0109 (0.01236690032280098) + + + --> TIME: 2024-10-19 23:26:15 -- STEP: 5950/69059 -- GLOBAL_STEP: 5950 + | > loss_text_ce: 0.04668958857655525 (0.07007745433317021) + | > loss_mel_ce: 3.6437621116638184 (3.9035765107739753) + | > loss: 0.9226129055023193 (0.9934134911689416) + | > current_lr: 5e-06 + | > step_time: 0.3115 (0.2445431561830665) + | > loader_time: 0.0126 (0.012360444830245336) + + + --> TIME: 2024-10-19 23:26:34 -- STEP: 6000/69059 -- GLOBAL_STEP: 6000 + | > loss_text_ce: 0.04579061269760132 (0.06987776873757445) + | > loss_mel_ce: 3.722524642944336 (3.9014664189418036) + | > loss: 0.9420788288116455 (0.9928360468546529) + | > current_lr: 5e-06 + | > step_time: 0.2595 (0.24453728048006693) + | > loader_time: 0.0112 (0.012357257723808293) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_6000.pth + + --> TIME: 2024-10-19 23:27:08 -- STEP: 6050/69059 -- GLOBAL_STEP: 6050 + | > loss_text_ce: 0.04700441285967827 (0.06968296342583243) + | > loss_mel_ce: 3.457151412963867 (3.8999625915732143) + | > loss: 0.8760389685630798 (0.9924113887302125) + | > current_lr: 5e-06 + | > step_time: 0.2323 (0.24476394468102575) + | > loader_time: 0.012 (0.01235504891261582) + + + --> TIME: 2024-10-19 23:27:24 -- STEP: 6100/69059 -- GLOBAL_STEP: 6100 + | > loss_text_ce: 0.04311997815966606 (0.06948908238992343) + | > loss_mel_ce: 3.8130249977111816 (3.8978194394267853) + | > loss: 0.964036226272583 (0.9918271304153984) + | > current_lr: 5e-06 + | > step_time: 0.1734 (0.24449551828572008) + | > loader_time: 0.0125 (0.012348164456789615) + + + --> TIME: 2024-10-19 23:27:42 -- STEP: 6150/69059 -- GLOBAL_STEP: 6150 + | > loss_text_ce: 0.04192456603050232 (0.06929559383813925) + | > loss_mel_ce: 3.508786916732788 (3.8957404069590256) + | > loss: 0.8876778483390808 (0.9912590001269064) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.2443217627982783) + | > loader_time: 0.0115 (0.012342469634079357) + + + --> TIME: 2024-10-19 23:28:00 -- STEP: 6200/69059 -- GLOBAL_STEP: 6200 + | > loss_text_ce: 0.048550914973020554 (0.06910867377755138) + | > loss_mel_ce: 3.5667636394500732 (3.8937366564427536) + | > loss: 0.9038286209106445 (0.9907113324922879) + | > current_lr: 5e-06 + | > step_time: 0.1561 (0.24426366236902053) + | > loader_time: 0.0105 (0.012338266334226059) + + + --> TIME: 2024-10-19 23:28:18 -- STEP: 6250/69059 -- GLOBAL_STEP: 6250 + | > loss_text_ce: 0.04769261181354523 (0.06892542660653589) + | > loss_mel_ce: 3.740699291229248 (3.891763979606617) + | > loss: 0.9470979571342468 (0.9901723514842967) + | > current_lr: 5e-06 + | > step_time: 0.4499 (0.2443084061050415) + | > loader_time: 0.0135 (0.012334215278625493) + + + --> TIME: 2024-10-19 23:28:37 -- STEP: 6300/69059 -- GLOBAL_STEP: 6300 + | > loss_text_ce: 0.04610591381788254 (0.06873991907056835) + | > loss_mel_ce: 3.4662506580352783 (3.889485133034831) + | > loss: 0.8780891299247742 (0.9895562629472622) + | > current_lr: 5e-06 + | > step_time: 0.1822 (0.24430375360307238) + | > loader_time: 0.0114 (0.012331158093043741) + + + --> TIME: 2024-10-19 23:28:55 -- STEP: 6350/69059 -- GLOBAL_STEP: 6350 + | > loss_text_ce: 0.041086167097091675 (0.06856211042427632) + | > loss_mel_ce: 3.315315008163452 (3.887623940715628) + | > loss: 0.8391003012657166 (0.9890465127295376) + | > current_lr: 5e-06 + | > step_time: 0.1938 (0.2442221394861777) + | > loader_time: 0.0105 (0.012323745404641463) + + + --> TIME: 2024-10-19 23:29:12 -- STEP: 6400/69059 -- GLOBAL_STEP: 6400 + | > loss_text_ce: 0.04931313544511795 (0.06838453449716317) + | > loss_mel_ce: 3.8413290977478027 (3.88516378443687) + | > loss: 0.9726605415344238 (0.9883870796766112) + | > current_lr: 5e-06 + | > step_time: 0.2997 (0.24407552666962146) + | > loader_time: 0.0125 (0.012316044829785829) + + + --> TIME: 2024-10-19 23:29:31 -- STEP: 6450/69059 -- GLOBAL_STEP: 6450 + | > loss_text_ce: 0.04723215475678444 (0.06820882438515961) + | > loss_mel_ce: 3.5518925189971924 (3.8834202639631408) + | > loss: 0.8997811675071716 (0.987907272041304) + | > current_lr: 5e-06 + | > step_time: 0.2015 (0.24419606097908908) + | > loader_time: 0.0109 (0.012311934722486397) + + + --> TIME: 2024-10-19 23:29:51 -- STEP: 6500/69059 -- GLOBAL_STEP: 6500 + | > loss_text_ce: 0.0427633598446846 (0.06803053772392167) + | > loss_mel_ce: 3.5654988288879395 (3.881629743135881) + | > loss: 0.9020655751228333 (0.9874150701486127) + | > current_lr: 5e-06 + | > step_time: 0.1588 (0.24425519572771512) + | > loader_time: 0.0115 (0.012306670482342064) + + + --> TIME: 2024-10-19 23:30:08 -- STEP: 6550/69059 -- GLOBAL_STEP: 6550 + | > loss_text_ce: 0.04080601781606674 (0.06785468573219891) + | > loss_mel_ce: 3.7283310890197754 (3.8797877883183025) + | > loss: 0.9422842860221863 (0.9869106184708234) + | > current_lr: 5e-06 + | > step_time: 0.3022 (0.24418569604859097) + | > loader_time: 0.0104 (0.01230165656286342) + + + --> TIME: 2024-10-19 23:30:30 -- STEP: 6600/69059 -- GLOBAL_STEP: 6600 + | > loss_text_ce: 0.04894663766026497 (0.06768483669920392) + | > loss_mel_ce: 4.085399150848389 (3.87810185154278) + | > loss: 1.0335865020751953 (0.9864466720458214) + | > current_lr: 5e-06 + | > step_time: 0.5032 (0.24451976241487444) + | > loader_time: 0.0171 (0.012300907156684186) + + + --> TIME: 2024-10-19 23:30:47 -- STEP: 6650/69059 -- GLOBAL_STEP: 6650 + | > loss_text_ce: 0.0421689972281456 (0.06751448191478283) + | > loss_mel_ce: 3.648146152496338 (3.8761289292529004) + | > loss: 0.9225788116455078 (0.98591085281587) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.24435716754511783) + | > loader_time: 0.0105 (0.012292691926310837) + + + --> TIME: 2024-10-19 23:31:07 -- STEP: 6700/69059 -- GLOBAL_STEP: 6700 + | > loss_text_ce: 0.044055044651031494 (0.06734749449889599) + | > loss_mel_ce: 3.4476704597473145 (3.874652271306326) + | > loss: 0.8729313611984253 (0.9854999414753539) + | > current_lr: 5e-06 + | > step_time: 0.1635 (0.24449581687130145) + | > loader_time: 0.0105 (0.01228860513487859) + + + --> TIME: 2024-10-19 23:31:27 -- STEP: 6750/69059 -- GLOBAL_STEP: 6750 + | > loss_text_ce: 0.04564967378973961 (0.06718662893937688) + | > loss_mel_ce: 3.3037109375 (3.8729368880942907) + | > loss: 0.8373401761054993 (0.9850308792944289) + | > current_lr: 5e-06 + | > step_time: 0.1861 (0.24464693835929588) + | > loader_time: 0.0107 (0.012284894201490619) + + + --> TIME: 2024-10-19 23:31:46 -- STEP: 6800/69059 -- GLOBAL_STEP: 6800 + | > loss_text_ce: 0.04333215951919556 (0.06702408879228373) + | > loss_mel_ce: 3.3383290767669678 (3.8712172083994814) + | > loss: 0.8454152941703796 (0.9845603243305384) + | > current_lr: 5e-06 + | > step_time: 0.197 (0.24473446428775789) + | > loader_time: 0.0115 (0.012281642906806053) + + + --> TIME: 2024-10-19 23:32:05 -- STEP: 6850/69059 -- GLOBAL_STEP: 6850 + | > loss_text_ce: 0.04097823053598404 (0.06686114448428587) + | > loss_mel_ce: 3.604278802871704 (3.869468541423759) + | > loss: 0.9113142490386963 (0.9840824215168482) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.24484136108064303) + | > loader_time: 0.0102 (0.012277017022571429) + + + --> TIME: 2024-10-19 23:32:23 -- STEP: 6900/69059 -- GLOBAL_STEP: 6900 + | > loss_text_ce: 0.044486284255981445 (0.06670636140403974) + | > loss_mel_ce: 3.4201369285583496 (3.867370457856541) + | > loss: 0.8661558032035828 (0.9835192048808784) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.24475595121798308) + | > loader_time: 0.0102 (0.01229287817858268) + + + --> TIME: 2024-10-19 23:32:42 -- STEP: 6950/69059 -- GLOBAL_STEP: 6950 + | > loss_text_ce: 0.04274147003889084 (0.06654823192107168) + | > loss_mel_ce: 3.641183853149414 (3.8658858659455997) + | > loss: 0.9209813475608826 (0.9831085245300522) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.2447563711344767) + | > loader_time: 0.0102 (0.01228672545590847) + + + --> TIME: 2024-10-19 23:33:00 -- STEP: 7000/69059 -- GLOBAL_STEP: 7000 + | > loss_text_ce: 0.04453542083501816 (0.06639588448139179) + | > loss_mel_ce: 3.769951581954956 (3.863976668494078) + | > loss: 0.9536217451095581 (0.9825931382860438) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.24473269142423357) + | > loader_time: 0.0105 (0.012280755553926744) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_7000.pth + + --> TIME: 2024-10-19 23:33:32 -- STEP: 7050/69059 -- GLOBAL_STEP: 7050 + | > loss_text_ce: 0.04308898374438286 (0.06623815359693049) + | > loss_mel_ce: 3.5278594493865967 (3.862061944278409) + | > loss: 0.892737090587616 (0.9820750245473046) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.24461951756308264) + | > loader_time: 0.0108 (0.012277645320757065) + + + --> TIME: 2024-10-19 23:33:52 -- STEP: 7100/69059 -- GLOBAL_STEP: 7100 + | > loss_text_ce: 0.0462530143558979 (0.0660897054680636) + | > loss_mel_ce: 3.361520290374756 (3.8605789502909382) + | > loss: 0.85194331407547 (0.9816671640100596) + | > current_lr: 5e-06 + | > step_time: 0.1634 (0.24471685308805655) + | > loader_time: 0.011 (0.01227810896618266) + + + --> TIME: 2024-10-19 23:34:10 -- STEP: 7150/69059 -- GLOBAL_STEP: 7150 + | > loss_text_ce: 0.04326310381293297 (0.06594376426737207) + | > loss_mel_ce: 3.2726268768310547 (3.8587118528939524) + | > loss: 0.8289725184440613 (0.9811639043667916) + | > current_lr: 5e-06 + | > step_time: 0.2352 (0.24464552412499915) + | > loader_time: 0.0116 (0.01227472935523187) + + + --> TIME: 2024-10-19 23:34:28 -- STEP: 7200/69059 -- GLOBAL_STEP: 7200 + | > loss_text_ce: 0.04871940612792969 (0.06579566377525525) + | > loss_mel_ce: 3.772019624710083 (3.8569140707121856) + | > loss: 0.9551847577095032 (0.9806774337341371) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.24466377086109586) + | > loader_time: 0.0109 (0.012274009154902569) + + + --> TIME: 2024-10-19 23:34:48 -- STEP: 7250/69059 -- GLOBAL_STEP: 7250 + | > loss_text_ce: 0.04708995297551155 (0.06565190353989594) + | > loss_mel_ce: 3.516875982284546 (3.8552602055319323) + | > loss: 0.8909915089607239 (0.9802280273684122) + | > current_lr: 5e-06 + | > step_time: 0.2162 (0.24475365661752635) + | > loader_time: 0.0111 (0.012272044543562267) + + + --> TIME: 2024-10-19 23:35:06 -- STEP: 7300/69059 -- GLOBAL_STEP: 7300 + | > loss_text_ce: 0.04314117506146431 (0.06550148274545385) + | > loss_mel_ce: 3.617910385131836 (3.853658842419915) + | > loss: 0.9152628779411316 (0.9797900813981261) + | > current_lr: 5e-06 + | > step_time: 0.1633 (0.24475578161135111) + | > loader_time: 0.0108 (0.012269249158362824) + + + --> TIME: 2024-10-19 23:35:25 -- STEP: 7350/69059 -- GLOBAL_STEP: 7350 + | > loss_text_ce: 0.042971815913915634 (0.06536306512649764) + | > loss_mel_ce: 3.6213862895965576 (3.8523307989730218) + | > loss: 0.9160895347595215 (0.9794234661342318) + | > current_lr: 5e-06 + | > step_time: 0.194 (0.24480213895135997) + | > loader_time: 0.0109 (0.01226771821781081) + + + --> TIME: 2024-10-19 23:35:44 -- STEP: 7400/69059 -- GLOBAL_STEP: 7400 + | > loss_text_ce: 0.03953845053911209 (0.06522209533364375) + | > loss_mel_ce: 3.3068737983703613 (3.8504204234883495) + | > loss: 0.836603045463562 (0.9789106297815151) + | > current_lr: 5e-06 + | > step_time: 0.1563 (0.24484407924317025) + | > loader_time: 0.011 (0.01226633033236942) + + + --> TIME: 2024-10-19 23:36:02 -- STEP: 7450/69059 -- GLOBAL_STEP: 7450 + | > loss_text_ce: 0.042641639709472656 (0.0650830579709085) + | > loss_mel_ce: 3.7938992977142334 (3.8485702683621508) + | > loss: 0.9591352343559265 (0.9784133316686472) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.24475802927209228) + | > loader_time: 0.0106 (0.012262284323673125) + + + --> TIME: 2024-10-19 23:36:20 -- STEP: 7500/69059 -- GLOBAL_STEP: 7500 + | > loss_text_ce: 0.045516833662986755 (0.06494504324148082) + | > loss_mel_ce: 3.8596715927124023 (3.8469488467852178) + | > loss: 0.9762970805168152 (0.9779734725952132) + | > current_lr: 5e-06 + | > step_time: 0.2517 (0.24470414339701335) + | > loader_time: 0.0172 (0.012259205087025963) + + + --> TIME: 2024-10-19 23:36:39 -- STEP: 7550/69059 -- GLOBAL_STEP: 7550 + | > loss_text_ce: 0.040465254336595535 (0.06480634279993179) + | > loss_mel_ce: 3.467268943786621 (3.8455212823602443) + | > loss: 0.8769335746765137 (0.9775819063739255) + | > current_lr: 5e-06 + | > step_time: 0.1938 (0.24474242166178117) + | > loader_time: 0.0113 (0.01225756774674978) + + + --> TIME: 2024-10-19 23:36:57 -- STEP: 7600/69059 -- GLOBAL_STEP: 7600 + | > loss_text_ce: 0.03949649631977081 (0.06466850566187564) + | > loss_mel_ce: 3.5714709758758545 (3.844028663666615) + | > loss: 0.9027418494224548 (0.9771742923949878) + | > current_lr: 5e-06 + | > step_time: 0.163 (0.24467177921219876) + | > loader_time: 0.0109 (0.012253985750047788) + + + --> TIME: 2024-10-19 23:37:17 -- STEP: 7650/69059 -- GLOBAL_STEP: 7650 + | > loss_text_ce: 0.044681087136268616 (0.06453743170922467) + | > loss_mel_ce: 3.4907138347625732 (3.842556555442551) + | > loss: 0.8838487267494202 (0.9767734968740159) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24481563991970487) + | > loader_time: 0.0107 (0.012253077949573794) + + + --> TIME: 2024-10-19 23:37:35 -- STEP: 7700/69059 -- GLOBAL_STEP: 7700 + | > loss_text_ce: 0.04146948084235191 (0.06440054876705634) + | > loss_mel_ce: 3.544020175933838 (3.840794715819411) + | > loss: 0.8963724374771118 (0.9762988162582554) + | > current_lr: 5e-06 + | > step_time: 0.1714 (0.24477147579193115) + | > loader_time: 0.0109 (0.012248484592933163) + + + --> TIME: 2024-10-19 23:37:55 -- STEP: 7750/69059 -- GLOBAL_STEP: 7750 + | > loss_text_ce: 0.043381862342357635 (0.0642705692740215) + | > loss_mel_ce: 3.6381514072418213 (3.8394006697439282) + | > loss: 0.9203833341598511 (0.9759178098709336) + | > current_lr: 5e-06 + | > step_time: 0.2049 (0.2449405592333886) + | > loader_time: 0.0107 (0.012246716806965494) + + + --> TIME: 2024-10-19 23:38:13 -- STEP: 7800/69059 -- GLOBAL_STEP: 7800 + | > loss_text_ce: 0.04125678911805153 (0.06413949930610538) + | > loss_mel_ce: 3.4476890563964844 (3.8375733538468584) + | > loss: 0.8722364902496338 (0.9754282134083585) + | > current_lr: 5e-06 + | > step_time: 0.1515 (0.24488465917416108) + | > loader_time: 0.0102 (0.012240870854793454) + + + --> TIME: 2024-10-19 23:38:31 -- STEP: 7850/69059 -- GLOBAL_STEP: 7850 + | > loss_text_ce: 0.04587246850132942 (0.06400801724499167) + | > loss_mel_ce: 3.6828224658966064 (3.8360487431933072) + | > loss: 0.9321737289428711 (0.9750141902486213) + | > current_lr: 5e-06 + | > step_time: 0.2061 (0.24480710703855868) + | > loader_time: 0.0106 (0.012234641488190673) + + + --> TIME: 2024-10-19 23:38:48 -- STEP: 7900/69059 -- GLOBAL_STEP: 7900 + | > loss_text_ce: 0.04514467716217041 (0.0638802036982545) + | > loss_mel_ce: 3.4373092651367188 (3.834315857313847) + | > loss: 0.8706134557723999 (0.9745490153756307) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.24472216488439827) + | > loader_time: 0.0103 (0.012227867343757729) + + + --> TIME: 2024-10-19 23:39:07 -- STEP: 7950/69059 -- GLOBAL_STEP: 7950 + | > loss_text_ce: 0.04534624516963959 (0.06375377054999798) + | > loss_mel_ce: 3.4473166465759277 (3.8325194356126513) + | > loss: 0.8731657266616821 (0.9740683016837002) + | > current_lr: 5e-06 + | > step_time: 0.185 (0.24474687765229422) + | > loader_time: 0.0105 (0.012224127181670954) + + + --> TIME: 2024-10-19 23:39:26 -- STEP: 8000/69059 -- GLOBAL_STEP: 8000 + | > loss_text_ce: 0.04407607391476631 (0.06362740961182828) + | > loss_mel_ce: 3.5074408054351807 (3.8308763552904037) + | > loss: 0.8878791928291321 (0.9736259413883074) + | > current_lr: 5e-06 + | > step_time: 0.1975 (0.2447731242775917) + | > loader_time: 0.0113 (0.012221101522445682) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_8000.pth + + --> TIME: 2024-10-19 23:39:57 -- STEP: 8050/69059 -- GLOBAL_STEP: 8050 + | > loss_text_ce: 0.04192051663994789 (0.06350318437499992) + | > loss_mel_ce: 3.7138164043426514 (3.8293137189320063) + | > loss: 0.9389342069625854 (0.9732042259977455) + | > current_lr: 5e-06 + | > step_time: 0.232 (0.2447667706531027) + | > loader_time: 0.0116 (0.01221974577222552) + + + --> TIME: 2024-10-19 23:40:16 -- STEP: 8100/69059 -- GLOBAL_STEP: 8100 + | > loss_text_ce: 0.03970234841108322 (0.06338058557729258) + | > loss_mel_ce: 3.385772228240967 (3.827522081828403) + | > loss: 0.8563686609268188 (0.9727256670189477) + | > current_lr: 5e-06 + | > step_time: 0.1564 (0.2447462237617116) + | > loader_time: 0.012 (0.012216314769085546) + + + --> TIME: 2024-10-19 23:40:36 -- STEP: 8150/69059 -- GLOBAL_STEP: 8150 + | > loss_text_ce: 0.04705018922686577 (0.06326097547505129) + | > loss_mel_ce: 3.6801469326019287 (3.8264757124777864) + | > loss: 0.9317992925643921 (0.972434172154934) + | > current_lr: 5e-06 + | > step_time: 0.4918 (0.24486296007238284) + | > loader_time: 0.0141 (0.012217476528846416) + + + --> TIME: 2024-10-19 23:40:55 -- STEP: 8200/69059 -- GLOBAL_STEP: 8200 + | > loss_text_ce: 0.04259049892425537 (0.06314135486350882) + | > loss_mel_ce: 3.6200551986694336 (3.8253275632858186) + | > loss: 0.9156614542007446 (0.972117229700087) + | > current_lr: 5e-06 + | > step_time: 0.1519 (0.24492872156747958) + | > loader_time: 0.0122 (0.012217589704001825) + + + --> TIME: 2024-10-19 23:41:14 -- STEP: 8250/69059 -- GLOBAL_STEP: 8250 + | > loss_text_ce: 0.04830927401781082 (0.06302225863165892) + | > loss_mel_ce: 3.940288543701172 (3.8237201396768654) + | > loss: 0.9971494674682617 (0.9716855997461246) + | > current_lr: 5e-06 + | > step_time: 0.5292 (0.24496656984271428) + | > loader_time: 0.0152 (0.012215908888614544) + + + --> TIME: 2024-10-19 23:41:33 -- STEP: 8300/69059 -- GLOBAL_STEP: 8300 + | > loss_text_ce: 0.04574989527463913 (0.06290275966950533) + | > loss_mel_ce: 4.243669033050537 (3.8225361639907476) + | > loss: 1.0723546743392944 (0.9713597310882002) + | > current_lr: 5e-06 + | > step_time: 0.3592 (0.24499622951070948) + | > loader_time: 0.0128 (0.012216039565672363) + + + --> TIME: 2024-10-19 23:41:51 -- STEP: 8350/69059 -- GLOBAL_STEP: 8350 + | > loss_text_ce: 0.03872062265872955 (0.06278747065918493) + | > loss_mel_ce: 3.4538702964782715 (3.8209050731887277) + | > loss: 0.87314772605896 (0.9709231361443396) + | > current_lr: 5e-06 + | > step_time: 0.2117 (0.24494176987402455) + | > loader_time: 0.0114 (0.01221173383518608) + + + --> TIME: 2024-10-19 23:42:10 -- STEP: 8400/69059 -- GLOBAL_STEP: 8400 + | > loss_text_ce: 0.042789116501808167 (0.06267073761937836) + | > loss_mel_ce: 3.46811842918396 (3.819594741009522) + | > loss: 0.8777269124984741 (0.9705663698273029) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24501215381281716) + | > loader_time: 0.0114 (0.012209784899439137) + + + --> TIME: 2024-10-19 23:42:28 -- STEP: 8450/69059 -- GLOBAL_STEP: 8450 + | > loss_text_ce: 0.047195930033922195 (0.06255681682444145) + | > loss_mel_ce: 3.8422186374664307 (3.818079943092608) + | > loss: 0.9723536372184753 (0.970159190152522) + | > current_lr: 5e-06 + | > step_time: 0.3416 (0.24499871598192927) + | > loader_time: 0.0126 (0.012206048034351961) + + + --> TIME: 2024-10-19 23:42:46 -- STEP: 8500/69059 -- GLOBAL_STEP: 8500 + | > loss_text_ce: 0.046116285026073456 (0.062444120733614855) + | > loss_mel_ce: 3.7663211822509766 (3.8165052774092634) + | > loss: 0.9531093835830688 (0.9697373497135482) + | > current_lr: 5e-06 + | > step_time: 0.2265 (0.24494710661383237) + | > loader_time: 0.0118 (0.01220192600699032) + + + --> TIME: 2024-10-19 23:43:04 -- STEP: 8550/69059 -- GLOBAL_STEP: 8550 + | > loss_text_ce: 0.039078377187252045 (0.062331851011479436) + | > loss_mel_ce: 3.5506608486175537 (3.814955914675832) + | > loss: 0.8974348306655884 (0.9693219416043894) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.24490194705494664) + | > loader_time: 0.0104 (0.012198728650634046) + + + --> TIME: 2024-10-19 23:43:22 -- STEP: 8600/69059 -- GLOBAL_STEP: 8600 + | > loss_text_ce: 0.042401302605867386 (0.06221821629030723) + | > loss_mel_ce: 3.5982067584991455 (3.813502504132505) + | > loss: 0.910152018070221 (0.9689301802876367) + | > current_lr: 5e-06 + | > step_time: 0.1644 (0.24486230711604273) + | > loader_time: 0.0104 (0.012195536646732055) + + + --> TIME: 2024-10-19 23:43:41 -- STEP: 8650/69059 -- GLOBAL_STEP: 8650 + | > loss_text_ce: 0.045602768659591675 (0.06210617770263189) + | > loss_mel_ce: 3.701561450958252 (3.8123460424290814) + | > loss: 0.9367910623550415 (0.9686130552278072) + | > current_lr: 5e-06 + | > step_time: 0.3634 (0.24489300854633309) + | > loader_time: 0.0129 (0.012191547465462204) + + + --> TIME: 2024-10-19 23:44:01 -- STEP: 8700/69059 -- GLOBAL_STEP: 8700 + | > loss_text_ce: 0.04102310910820961 (0.06199776754768036) + | > loss_mel_ce: 3.5410573482513428 (3.8112795177821357) + | > loss: 0.8955200910568237 (0.9683193215488001) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.2449644907315572) + | > loader_time: 0.0107 (0.0121895295450057) + + + --> TIME: 2024-10-19 23:44:19 -- STEP: 8750/69059 -- GLOBAL_STEP: 8750 + | > loss_text_ce: 0.04257720708847046 (0.0618905640546761) + | > loss_mel_ce: 3.6309502124786377 (3.8097497124535598) + | > loss: 0.9183818697929382 (0.9679100693430203) + | > current_lr: 5e-06 + | > step_time: 0.2954 (0.24497167426518032) + | > loader_time: 0.0115 (0.0121844763619559) + + + --> TIME: 2024-10-19 23:44:38 -- STEP: 8800/69059 -- GLOBAL_STEP: 8800 + | > loss_text_ce: 0.041801273822784424 (0.0617804232379418) + | > loss_mel_ce: 3.6120858192443848 (3.8083602142875748) + | > loss: 0.9134717583656311 (0.9675351595810852) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.2449804162979126) + | > loader_time: 0.0103 (0.012181084860454891) + + + --> TIME: 2024-10-19 23:44:57 -- STEP: 8850/69059 -- GLOBAL_STEP: 8850 + | > loss_text_ce: 0.04642954096198082 (0.06167407012760266) + | > loss_mel_ce: 3.421220541000366 (3.806880278344872) + | > loss: 0.8669125437736511 (0.9671385872970176) + | > current_lr: 5e-06 + | > step_time: 0.2668 (0.24499225823892712) + | > loader_time: 0.0115 (0.012176459522570574) + + + --> TIME: 2024-10-19 23:45:15 -- STEP: 8900/69059 -- GLOBAL_STEP: 8900 + | > loss_text_ce: 0.040242522954940796 (0.06156844672569028) + | > loss_mel_ce: 3.4228503704071045 (3.805648784985693) + | > loss: 0.8657732009887695 (0.9668043081278195) + | > current_lr: 5e-06 + | > step_time: 0.1527 (0.2449789944123686) + | > loader_time: 0.0103 (0.012172464279646267) + + + --> TIME: 2024-10-19 23:45:34 -- STEP: 8950/69059 -- GLOBAL_STEP: 8950 + | > loss_text_ce: 0.043655481189489365 (0.06146202580389328) + | > loss_mel_ce: 3.8216607570648193 (3.8042630137278284) + | > loss: 0.966329038143158 (0.966431260088967) + | > current_lr: 5e-06 + | > step_time: 0.225 (0.24498430622356565) + | > loader_time: 0.0126 (0.012169954923278105) + + + --> TIME: 2024-10-19 23:45:52 -- STEP: 9000/69059 -- GLOBAL_STEP: 9000 + | > loss_text_ce: 0.04843208193778992 (0.06136060403452944) + | > loss_mel_ce: 3.3619821071624756 (3.802855168845909) + | > loss: 0.852603554725647 (0.9660539434353494) + | > current_lr: 5e-06 + | > step_time: 0.1575 (0.24493473148345948) + | > loader_time: 0.0109 (0.012167112270990989) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_9000.pth + + --> TIME: 2024-10-19 23:46:24 -- STEP: 9050/69059 -- GLOBAL_STEP: 9050 + | > loss_text_ce: 0.043031543493270874 (0.06126000560531924) + | > loss_mel_ce: 3.4386801719665527 (3.801551818426133) + | > loss: 0.8704279065132141 (0.9657029562365268) + | > current_lr: 5e-06 + | > step_time: 0.217 (0.2449622814167929) + | > loader_time: 0.0118 (0.012164447189035981) + + + --> TIME: 2024-10-19 23:46:42 -- STEP: 9100/69059 -- GLOBAL_STEP: 9100 + | > loss_text_ce: 0.042099952697753906 (0.06115747745670753) + | > loss_mel_ce: 3.485382556915283 (3.800039394635411) + | > loss: 0.8818706274032593 (0.9652992182380534) + | > current_lr: 5e-06 + | > step_time: 0.1528 (0.24489031482528853) + | > loader_time: 0.0105 (0.01216411121598966) + + + --> TIME: 2024-10-19 23:47:02 -- STEP: 9150/69059 -- GLOBAL_STEP: 9150 + | > loss_text_ce: 0.03955880180001259 (0.061055665130171924) + | > loss_mel_ce: 3.1554019451141357 (3.7987079803800228) + | > loss: 0.7987402081489563 (0.9649409116291593) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.2450222226700496) + | > loader_time: 0.0107 (0.012163925431465186) + + + --> TIME: 2024-10-19 23:47:20 -- STEP: 9200/69059 -- GLOBAL_STEP: 9200 + | > loss_text_ce: 0.04315585643053055 (0.060952575121722447) + | > loss_mel_ce: 3.341841220855713 (3.7974478419967226) + | > loss: 0.8462492823600769 (0.9646001045211486) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24494123756885527) + | > loader_time: 0.0106 (0.012163374164830083) + + + --> TIME: 2024-10-19 23:47:39 -- STEP: 9250/69059 -- GLOBAL_STEP: 9250 + | > loss_text_ce: 0.043515756726264954 (0.06085158552752924) + | > loss_mel_ce: 3.4608545303344727 (3.7961041488389617) + | > loss: 0.8760925531387329 (0.9642389338274244) + | > current_lr: 5e-06 + | > step_time: 0.3667 (0.24497973246187776) + | > loader_time: 0.0125 (0.012161355637215272) + + + --> TIME: 2024-10-19 23:47:57 -- STEP: 9300/69059 -- GLOBAL_STEP: 9300 + | > loss_text_ce: 0.04052628576755524 (0.06075337553016246) + | > loss_mel_ce: 3.5908994674682617 (3.7946886063134704) + | > loss: 0.9078564643859863 (0.9638604956929386) + | > current_lr: 5e-06 + | > step_time: 0.1576 (0.24494599921728974) + | > loader_time: 0.0102 (0.012158318027373279) + + + --> TIME: 2024-10-19 23:48:16 -- STEP: 9350/69059 -- GLOBAL_STEP: 9350 + | > loss_text_ce: 0.04132341220974922 (0.060655764017831494) + | > loss_mel_ce: 3.6005311012268066 (3.7931862414711843) + | > loss: 0.9104636311531067 (0.9634605016172872) + | > current_lr: 5e-06 + | > step_time: 0.3458 (0.2450128492569541) + | > loader_time: 0.0129 (0.01215676300028428) + + + --> TIME: 2024-10-19 23:48:36 -- STEP: 9400/69059 -- GLOBAL_STEP: 9400 + | > loss_text_ce: 0.039002709090709686 (0.060558723815657016) + | > loss_mel_ce: 3.2931606769561768 (3.7920065182827796) + | > loss: 0.8330408334732056 (0.9631413107666548) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24515212193448493) + | > loader_time: 0.0109 (0.01215629859173551) + + + --> TIME: 2024-10-19 23:48:55 -- STEP: 9450/69059 -- GLOBAL_STEP: 9450 + | > loss_text_ce: 0.04579658433794975 (0.060463835418460524) + | > loss_mel_ce: 3.731351852416992 (3.790673172385595) + | > loss: 0.944287121295929 (0.9627842521982833) + | > current_lr: 5e-06 + | > step_time: 0.2355 (0.24517703909091848) + | > loader_time: 0.0119 (0.012155513637280336) + + + --> TIME: 2024-10-19 23:49:14 -- STEP: 9500/69059 -- GLOBAL_STEP: 9500 + | > loss_text_ce: 0.03896031156182289 (0.06036798185580622) + | > loss_mel_ce: 3.835742473602295 (3.7896791154961744) + | > loss: 0.9686756730079651 (0.9625117745838652) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.2451883434245461) + | > loader_time: 0.0109 (0.012154231573406015) + + + --> TIME: 2024-10-19 23:49:33 -- STEP: 9550/69059 -- GLOBAL_STEP: 9550 + | > loss_text_ce: 0.04449955001473427 (0.06027330478877576) + | > loss_mel_ce: 3.5450968742370605 (3.7883936096111426) + | > loss: 0.8973991274833679 (0.9621667288485607) + | > current_lr: 5e-06 + | > step_time: 0.1866 (0.24520321112028592) + | > loader_time: 0.0116 (0.012152255542615318) + + + --> TIME: 2024-10-19 23:49:52 -- STEP: 9600/69059 -- GLOBAL_STEP: 9600 + | > loss_text_ce: 0.04584953188896179 (0.060181897664442326) + | > loss_mel_ce: 3.673555612564087 (3.7873125433176664) + | > loss: 0.9298512935638428 (0.9618736104853436) + | > current_lr: 5e-06 + | > step_time: 0.1818 (0.24528115856150787) + | > loader_time: 0.0113 (0.012153750658035273) + + + --> TIME: 2024-10-19 23:50:11 -- STEP: 9650/69059 -- GLOBAL_STEP: 9650 + | > loss_text_ce: 0.03873113915324211 (0.06008670190030705) + | > loss_mel_ce: 3.417356252670288 (3.785815259459096) + | > loss: 0.8640218377113342 (0.9614754905787142) + | > current_lr: 5e-06 + | > step_time: 0.2807 (0.245261747725887) + | > loader_time: 0.0117 (0.012150925329929806) + + + --> TIME: 2024-10-19 23:50:29 -- STEP: 9700/69059 -- GLOBAL_STEP: 9700 + | > loss_text_ce: 0.04422665014863014 (0.05999582956324193) + | > loss_mel_ce: 3.285970449447632 (3.784548864807041) + | > loss: 0.8325492739677429 (0.9611361738517097) + | > current_lr: 5e-06 + | > step_time: 0.1639 (0.2452798123457997) + | > loader_time: 0.0103 (0.012148374331366153) + + + --> TIME: 2024-10-19 23:50:49 -- STEP: 9750/69059 -- GLOBAL_STEP: 9750 + | > loss_text_ce: 0.04325929656624794 (0.05990484213561551) + | > loss_mel_ce: 3.739064931869507 (3.7833237079229023) + | > loss: 0.9455810785293579 (0.9608071377949822) + | > current_lr: 5e-06 + | > step_time: 0.2806 (0.24534825031573956) + | > loader_time: 0.0119 (0.012146588325500484) + + + --> TIME: 2024-10-19 23:51:06 -- STEP: 9800/69059 -- GLOBAL_STEP: 9800 + | > loss_text_ce: 0.04038252308964729 (0.05981528182799078) + | > loss_mel_ce: 3.4493179321289062 (3.7820386337017435) + | > loss: 0.8724251389503479 (0.9604634791673431) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.2452420597417014) + | > loader_time: 0.011 (0.012142399573812678) + + + --> TIME: 2024-10-19 23:51:25 -- STEP: 9850/69059 -- GLOBAL_STEP: 9850 + | > loss_text_ce: 0.0424397811293602 (0.059722712133363096) + | > loss_mel_ce: 3.7893590927124023 (3.780851289366699) + | > loss: 0.957949697971344 (0.9601435006572495) + | > current_lr: 5e-06 + | > step_time: 0.4067 (0.24527167763201718) + | > loader_time: 0.0148 (0.0121410892215477) + + + --> TIME: 2024-10-19 23:51:44 -- STEP: 9900/69059 -- GLOBAL_STEP: 9900 + | > loss_text_ce: 0.04468762129545212 (0.05963286071789955) + | > loss_mel_ce: 3.4993152618408203 (3.7796458671068818) + | > loss: 0.8860006928443909 (0.959819682234464) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.2452818364567227) + | > loader_time: 0.0115 (0.012139767324081568) + + + --> TIME: 2024-10-19 23:52:04 -- STEP: 9950/69059 -- GLOBAL_STEP: 9950 + | > loss_text_ce: 0.04350562393665314 (0.05954754614687713) + | > loss_mel_ce: 3.3680267333984375 (3.778539355579923) + | > loss: 0.8528831005096436 (0.959521725716901) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.24538770222783687) + | > loader_time: 0.0116 (0.012139033121080247) + + + --> TIME: 2024-10-19 23:52:22 -- STEP: 10000/69059 -- GLOBAL_STEP: 10000 + | > loss_text_ce: 0.04345876723527908 (0.05946288323812155) + | > loss_mel_ce: 3.4265124797821045 (3.777461177849761) + | > loss: 0.8674927949905396 (0.9592310155570493) + | > current_lr: 5e-06 + | > step_time: 0.1709 (0.2454047972679138) + | > loader_time: 0.0105 (0.012135841941833488) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_10000.pth + + --> TIME: 2024-10-19 23:52:54 -- STEP: 10050/69059 -- GLOBAL_STEP: 10050 + | > loss_text_ce: 0.04292137175798416 (0.059375523494843506) + | > loss_mel_ce: 3.724569320678711 (3.776030905306034) + | > loss: 0.9418726563453674 (0.9588516074863818) + | > current_lr: 5e-06 + | > step_time: 0.3225 (0.24540138659785635) + | > loader_time: 0.0123 (0.012133821231215739) + + + --> TIME: 2024-10-19 23:53:13 -- STEP: 10100/69059 -- GLOBAL_STEP: 10100 + | > loss_text_ce: 0.04060046002268791 (0.05929266226579321) + | > loss_mel_ce: 3.4139885902404785 (3.775037988837394) + | > loss: 0.8636472821235657 (0.9585826630521515) + | > current_lr: 5e-06 + | > step_time: 0.1779 (0.24536839761356316) + | > loader_time: 0.0108 (0.012131309013555537) + + + --> TIME: 2024-10-19 23:53:31 -- STEP: 10150/69059 -- GLOBAL_STEP: 10150 + | > loss_text_ce: 0.03938370943069458 (0.059208043602666516) + | > loss_mel_ce: 3.53224778175354 (3.7737127937589285) + | > loss: 0.8929078578948975 (0.9582302096148415) + | > current_lr: 5e-06 + | > step_time: 0.3363 (0.24535989272770622) + | > loader_time: 0.0121 (0.01212864906329826) + + + --> TIME: 2024-10-19 23:53:49 -- STEP: 10200/69059 -- GLOBAL_STEP: 10200 + | > loss_text_ce: 0.04621078446507454 (0.05912355446092315) + | > loss_mel_ce: 3.694969892501831 (3.7726296961541186) + | > loss: 0.9352951645851135 (0.9579383128998312) + | > current_lr: 5e-06 + | > step_time: 0.2795 (0.24533703649745267) + | > loader_time: 0.0113 (0.012127626339594516) + + + --> TIME: 2024-10-19 23:54:08 -- STEP: 10250/69059 -- GLOBAL_STEP: 10250 + | > loss_text_ce: 0.04096240550279617 (0.059038695283778975) + | > loss_mel_ce: 3.426175117492676 (3.77158412949631) + | > loss: 0.866784393787384 (0.9576557064521589) + | > current_lr: 5e-06 + | > step_time: 0.2779 (0.24532155569588265) + | > loader_time: 0.012 (0.012125076061341816) + + + --> TIME: 2024-10-19 23:54:26 -- STEP: 10300/69059 -- GLOBAL_STEP: 10300 + | > loss_text_ce: 0.04157039895653725 (0.0589533845233972) + | > loss_mel_ce: 3.4798245429992676 (3.770397370486575) + | > loss: 0.8803487420082092 (0.9573376890004246) + | > current_lr: 5e-06 + | > step_time: 0.2454 (0.24531005556143604) + | > loader_time: 0.0119 (0.012122545358046742) + + + --> TIME: 2024-10-19 23:54:45 -- STEP: 10350/69059 -- GLOBAL_STEP: 10350 + | > loss_text_ce: 0.04081248492002487 (0.058868776547591734) + | > loss_mel_ce: 3.367934465408325 (3.7690996481715686) + | > loss: 0.8521867394447327 (0.9569921064434406) + | > current_lr: 5e-06 + | > step_time: 0.2228 (0.24529471084115587) + | > loader_time: 0.012 (0.01212177898572838) + + + --> TIME: 2024-10-19 23:55:04 -- STEP: 10400/69059 -- GLOBAL_STEP: 10400 + | > loss_text_ce: 0.0406377799808979 (0.058785397460540595) + | > loss_mel_ce: 3.635781764984131 (3.7680250018605737) + | > loss: 0.9191048741340637 (0.9567026001100343) + | > current_lr: 5e-06 + | > step_time: 0.2969 (0.24531942885655622) + | > loader_time: 0.0123 (0.012119591465363132) + + + --> TIME: 2024-10-19 23:55:22 -- STEP: 10450/69059 -- GLOBAL_STEP: 10450 + | > loss_text_ce: 0.039759278297424316 (0.05870310698150132) + | > loss_mel_ce: 3.181144952774048 (3.766784345134015) + | > loss: 0.8052260875701904 (0.9563718633218231) + | > current_lr: 5e-06 + | > step_time: 0.178 (0.24532410706059213) + | > loader_time: 0.0104 (0.012117894003836154) + + + --> TIME: 2024-10-19 23:55:41 -- STEP: 10500/69059 -- GLOBAL_STEP: 10500 + | > loss_text_ce: 0.04050690680742264 (0.05862215468571269) + | > loss_mel_ce: 3.6058921813964844 (3.765566547757095) + | > loss: 0.9115997552871704 (0.9560471759012753) + | > current_lr: 5e-06 + | > step_time: 0.2529 (0.24531525032860893) + | > loader_time: 0.0136 (0.012114506857735767) + + + --> TIME: 2024-10-19 23:55:58 -- STEP: 10550/69059 -- GLOBAL_STEP: 10550 + | > loss_text_ce: 0.04289042204618454 (0.05854152610340077) + | > loss_mel_ce: 3.6952145099639893 (3.764145374637071) + | > loss: 0.9345262050628662 (0.955671725459572) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.24528211378938214) + | > loader_time: 0.0105 (0.012112122870169533) + + + --> TIME: 2024-10-19 23:56:17 -- STEP: 10600/69059 -- GLOBAL_STEP: 10600 + | > loss_text_ce: 0.04136618226766586 (0.05846228676514213) + | > loss_mel_ce: 3.243013381958008 (3.762854927575805) + | > loss: 0.8210948705673218 (0.9553293038370461) + | > current_lr: 5e-06 + | > step_time: 0.1634 (0.24523534729795635) + | > loader_time: 0.0104 (0.012109738295932974) + + + --> TIME: 2024-10-19 23:56:35 -- STEP: 10650/69059 -- GLOBAL_STEP: 10650 + | > loss_text_ce: 0.038829825818538666 (0.058382236348211186) + | > loss_mel_ce: 3.4224774837493896 (3.761820224775387) + | > loss: 0.8653268218040466 (0.9550506155255801) + | > current_lr: 5e-06 + | > step_time: 0.2567 (0.24520785810801904) + | > loader_time: 0.0119 (0.012107804370038375) + + + --> TIME: 2024-10-19 23:56:52 -- STEP: 10700/69059 -- GLOBAL_STEP: 10700 + | > loss_text_ce: 0.03989453986287117 (0.058302213463270476) + | > loss_mel_ce: 3.3544092178344727 (3.7604177280229862) + | > loss: 0.8485759496688843 (0.9546799856201497) + | > current_lr: 5e-06 + | > step_time: 0.1558 (0.24511202665132897) + | > loader_time: 0.0105 (0.012104287080675631) + + + --> TIME: 2024-10-19 23:57:10 -- STEP: 10750/69059 -- GLOBAL_STEP: 10750 + | > loss_text_ce: 0.0385669581592083 (0.05822281101831141) + | > loss_mel_ce: 3.2665247917175293 (3.7591260294359703) + | > loss: 0.8262729644775391 (0.9543372103779801) + | > current_lr: 5e-06 + | > step_time: 0.1715 (0.2450657456198404) + | > loader_time: 0.0105 (0.012100357610125864) + + + --> TIME: 2024-10-19 23:57:29 -- STEP: 10800/69059 -- GLOBAL_STEP: 10800 + | > loss_text_ce: 0.04039851203560829 (0.058144504764220485) + | > loss_mel_ce: 3.2153704166412354 (3.7579659459325923) + | > loss: 0.8139422535896301 (0.9540276129378199) + | > current_lr: 5e-06 + | > step_time: 0.1573 (0.24511429919136896) + | > loader_time: 0.0103 (0.012097006969981713) + + + --> TIME: 2024-10-19 23:57:46 -- STEP: 10850/69059 -- GLOBAL_STEP: 10850 + | > loss_text_ce: 0.04313283413648605 (0.05806811828320722) + | > loss_mel_ce: 3.4180054664611816 (3.7567853422868005) + | > loss: 0.8652845621109009 (0.9537133654086807) + | > current_lr: 5e-06 + | > step_time: 0.185 (0.24499169433171847) + | > loader_time: 0.011 (0.012092145994511607) + + + --> TIME: 2024-10-19 23:58:05 -- STEP: 10900/69059 -- GLOBAL_STEP: 10900 + | > loss_text_ce: 0.0382336787879467 (0.05799089933685724) + | > loss_mel_ce: 3.449565887451172 (3.7557549168647895) + | > loss: 0.8719499111175537 (0.953436454321265) + | > current_lr: 5e-06 + | > step_time: 0.1658 (0.24505227121738118) + | > loader_time: 0.011 (0.012090720671032536) + + + --> TIME: 2024-10-19 23:58:23 -- STEP: 10950/69059 -- GLOBAL_STEP: 10950 + | > loss_text_ce: 0.04448255896568298 (0.05791592258296572) + | > loss_mel_ce: 3.892913341522217 (3.7545684516810858) + | > loss: 0.9843489527702332 (0.9531210938327376) + | > current_lr: 5e-06 + | > step_time: 0.3554 (0.24502834422403275) + | > loader_time: 0.0125 (0.012087444562345869) + + + --> TIME: 2024-10-19 23:58:43 -- STEP: 11000/69059 -- GLOBAL_STEP: 11000 + | > loss_text_ce: 0.044764965772628784 (0.057840382834726825) + | > loss_mel_ce: 3.852724313735962 (3.7534994443329817) + | > loss: 0.9743723273277283 (0.9528349570523595) + | > current_lr: 5e-06 + | > step_time: 0.3435 (0.24507058143615723) + | > loader_time: 0.0135 (0.012086064880544478) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_11000.pth + + --> TIME: 2024-10-19 23:59:16 -- STEP: 11050/69059 -- GLOBAL_STEP: 11050 + | > loss_text_ce: 0.04011791571974754 (0.05776484761066807) + | > loss_mel_ce: 3.4887213706970215 (3.752402522100035) + | > loss: 0.882209837436676 (0.9525418426925767) + | > current_lr: 5e-06 + | > step_time: 0.1872 (0.24507161192225116) + | > loader_time: 0.0109 (0.012085590189938081) + + + --> TIME: 2024-10-19 23:59:35 -- STEP: 11100/69059 -- GLOBAL_STEP: 11100 + | > loss_text_ce: 0.0402597114443779 (0.05769034483780435) + | > loss_mel_ce: 3.471696615219116 (3.7512376673801446) + | > loss: 0.8779890537261963 (0.9522320033247393) + | > current_lr: 5e-06 + | > step_time: 0.1657 (0.24509644796182442) + | > loader_time: 0.0112 (0.012085983581371125) + + + --> TIME: 2024-10-19 23:59:54 -- STEP: 11150/69059 -- GLOBAL_STEP: 11150 + | > loss_text_ce: 0.04042302444577217 (0.05761628145661026) + | > loss_mel_ce: 3.4135191440582275 (3.7501081630894877) + | > loss: 0.8634855151176453 (0.9519311114063165) + | > current_lr: 5e-06 + | > step_time: 0.2315 (0.2451449909979987) + | > loader_time: 0.011 (0.01208517666889412) + + + --> TIME: 2024-10-20 00:00:14 -- STEP: 11200/69059 -- GLOBAL_STEP: 11200 + | > loss_text_ce: 0.042591456323862076 (0.05754555911956593) + | > loss_mel_ce: 3.702993392944336 (3.749336468917975) + | > loss: 0.9363962411880493 (0.9517205072886165) + | > current_lr: 5e-06 + | > step_time: 0.2476 (0.24526727135692325) + | > loader_time: 0.0127 (0.01208484377179826) + + + --> TIME: 2024-10-20 00:00:32 -- STEP: 11250/69059 -- GLOBAL_STEP: 11250 + | > loss_text_ce: 0.047110527753829956 (0.05747299122710994) + | > loss_mel_ce: 3.2212064266204834 (3.7479389639536462) + | > loss: 0.8170792460441589 (0.9513529890802158) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24519678192138672) + | > loader_time: 0.0109 (0.012082804128858768) + + + --> TIME: 2024-10-20 00:00:50 -- STEP: 11300/69059 -- GLOBAL_STEP: 11300 + | > loss_text_ce: 0.04134140908718109 (0.05740025780402689) + | > loss_mel_ce: 3.6220409870147705 (3.746810098031973) + | > loss: 0.9158455729484558 (0.9510525892477107) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.2451495207094513) + | > loader_time: 0.0107 (0.012080325827134385) + + + --> TIME: 2024-10-20 00:01:08 -- STEP: 11350/69059 -- GLOBAL_STEP: 11350 + | > loss_text_ce: 0.037792228162288666 (0.05732771398540607) + | > loss_mel_ce: 3.6510846614837646 (3.74563131882755) + | > loss: 0.9222192168235779 (0.9507397585135712) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.24511064600839488) + | > loader_time: 0.0104 (0.012077905470054046) + + + --> TIME: 2024-10-20 00:01:27 -- STEP: 11400/69059 -- GLOBAL_STEP: 11400 + | > loss_text_ce: 0.04097326844930649 (0.05725653089844316) + | > loss_mel_ce: 3.343555450439453 (3.744710392470938) + | > loss: 0.8461321592330933 (0.9504917311563814) + | > current_lr: 5e-06 + | > step_time: 0.3308 (0.24517750556008858) + | > loader_time: 0.0131 (0.012076277983816037) + + + --> TIME: 2024-10-20 00:01:45 -- STEP: 11450/69059 -- GLOBAL_STEP: 11450 + | > loss_text_ce: 0.04286618158221245 (0.05718638364957642) + | > loss_mel_ce: 3.794593572616577 (3.743562460945155) + | > loss: 0.9593649506568909 (0.9501872114635438) + | > current_lr: 5e-06 + | > step_time: 0.2322 (0.24512972344477624) + | > loader_time: 0.0118 (0.012072680006901761) + + + --> TIME: 2024-10-20 00:02:03 -- STEP: 11500/69059 -- GLOBAL_STEP: 11500 + | > loss_text_ce: 0.04213527590036392 (0.05711416124746824) + | > loss_mel_ce: 3.753542184829712 (3.7423099257220316) + | > loss: 0.9489193558692932 (0.9498560220780567) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.2450713817762292) + | > loader_time: 0.0125 (0.01206864883588707) + + + --> TIME: 2024-10-20 00:02:22 -- STEP: 11550/69059 -- GLOBAL_STEP: 11550 + | > loss_text_ce: 0.0387495718896389 (0.05704360418053905) + | > loss_mel_ce: 3.8415510654449463 (3.7413764173127797) + | > loss: 0.9700751304626465 (0.9496050057008656) + | > current_lr: 5e-06 + | > step_time: 0.2671 (0.24512317025816285) + | > loader_time: 0.0116 (0.012067983470437837) + + + --> TIME: 2024-10-20 00:02:42 -- STEP: 11600/69059 -- GLOBAL_STEP: 11600 + | > loss_text_ce: 0.03995933756232262 (0.056975955950930954) + | > loss_mel_ce: 3.664931058883667 (3.7403602484382357) + | > loss: 0.9262226223945618 (0.9493340514343348) + | > current_lr: 5e-06 + | > step_time: 0.1626 (0.2451769812353726) + | > loader_time: 0.0102 (0.012066311055216286) + + + --> TIME: 2024-10-20 00:03:01 -- STEP: 11650/69059 -- GLOBAL_STEP: 11650 + | > loss_text_ce: 0.04277792200446129 (0.05690485815249772) + | > loss_mel_ce: 3.3208444118499756 (3.739444207899553) + | > loss: 0.8409056067466736 (0.9490872668503679) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.24519162779713904) + | > loader_time: 0.0108 (0.012065116407533561) + + + --> TIME: 2024-10-20 00:03:19 -- STEP: 11700/69059 -- GLOBAL_STEP: 11700 + | > loss_text_ce: 0.043457526713609695 (0.056839405061024105) + | > loss_mel_ce: 3.3406991958618164 (3.738175275488789) + | > loss: 0.8460391759872437 (0.9487536704693073) + | > current_lr: 5e-06 + | > step_time: 0.1641 (0.24517167447978613) + | > loader_time: 0.0105 (0.012063396649482912) + + + --> TIME: 2024-10-20 00:03:38 -- STEP: 11750/69059 -- GLOBAL_STEP: 11750 + | > loss_text_ce: 0.04354596510529518 (0.05676980256431887) + | > loss_mel_ce: 3.441098213195801 (3.7370667284295838) + | > loss: 0.8711610436439514 (0.9484591331076102) + | > current_lr: 5e-06 + | > step_time: 0.387 (0.24520253019129976) + | > loader_time: 0.0153 (0.012060999971754997) + + + --> TIME: 2024-10-20 00:03:57 -- STEP: 11800/69059 -- GLOBAL_STEP: 11800 + | > loss_text_ce: 0.042390916496515274 (0.05670385750673558) + | > loss_mel_ce: 3.4299285411834717 (3.7360173453718857) + | > loss: 0.8680798411369324 (0.9481803010631403) + | > current_lr: 5e-06 + | > step_time: 0.1975 (0.24521607542442064) + | > loader_time: 0.0125 (0.012059320676124693) + + + --> TIME: 2024-10-20 00:04:15 -- STEP: 11850/69059 -- GLOBAL_STEP: 11850 + | > loss_text_ce: 0.04031245782971382 (0.05663721409056482) + | > loss_mel_ce: 3.2566914558410645 (3.7350652911592563) + | > loss: 0.8242509961128235 (0.9479256266581849) + | > current_lr: 5e-06 + | > step_time: 0.1967 (0.2452058676828312) + | > loader_time: 0.0103 (0.012056189327803333) + + + --> TIME: 2024-10-20 00:04:33 -- STEP: 11900/69059 -- GLOBAL_STEP: 11900 + | > loss_text_ce: 0.03665778040885925 (0.0565727355969801) + | > loss_mel_ce: 3.682596206665039 (3.7339700609094884) + | > loss: 0.9298135042190552 (0.9476356994705027) + | > current_lr: 5e-06 + | > step_time: 0.1658 (0.24516303671508277) + | > loader_time: 0.0106 (0.012052348842139996) + + + --> TIME: 2024-10-20 00:04:51 -- STEP: 11950/69059 -- GLOBAL_STEP: 11950 + | > loss_text_ce: 0.043840307742357254 (0.056507717544625725) + | > loss_mel_ce: 3.6163330078125 (3.7329285573161224) + | > loss: 0.9150433540344238 (0.9473590690770396) + | > current_lr: 5e-06 + | > step_time: 0.2846 (0.24510951748452925) + | > loader_time: 0.012 (0.012050039119800255) + + + --> TIME: 2024-10-20 00:05:09 -- STEP: 12000/69059 -- GLOBAL_STEP: 12000 + | > loss_text_ce: 0.04309201240539551 (0.056440762140477434) + | > loss_mel_ce: 3.4902069568634033 (3.731932714839769) + | > loss: 0.8833247423171997 (0.94709336961309) + | > current_lr: 5e-06 + | > step_time: 0.1648 (0.24508066684007646) + | > loader_time: 0.0128 (0.012047702193260183) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_12000.pth + + --> TIME: 2024-10-20 00:05:42 -- STEP: 12050/69059 -- GLOBAL_STEP: 12050 + | > loss_text_ce: 0.04062905162572861 (0.05637651584795128) + | > loss_mel_ce: 3.6577975749969482 (3.731009514005345) + | > loss: 0.9246066808700562 (0.9468465078322212) + | > current_lr: 5e-06 + | > step_time: 0.2336 (0.24511381657786388) + | > loader_time: 0.0126 (0.012047882495579374) + + + --> TIME: 2024-10-20 00:06:02 -- STEP: 12100/69059 -- GLOBAL_STEP: 12100 + | > loss_text_ce: 0.04145808517932892 (0.056313429153100746) + | > loss_mel_ce: 3.908674716949463 (3.7301345523132694) + | > loss: 0.9875332117080688 (0.9466119957167243) + | > current_lr: 5e-06 + | > step_time: 0.281 (0.24519729846765187) + | > loader_time: 0.012 (0.01204929475941933) + + + --> TIME: 2024-10-20 00:06:21 -- STEP: 12150/69059 -- GLOBAL_STEP: 12150 + | > loss_text_ce: 0.03868549317121506 (0.056246225018007995) + | > loss_mel_ce: 3.44657826423645 (3.7291786355540513) + | > loss: 0.8713159561157227 (0.9463562154769886) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.24520474426049754) + | > loader_time: 0.0112 (0.01204874087753609) + + + --> TIME: 2024-10-20 00:06:39 -- STEP: 12200/69059 -- GLOBAL_STEP: 12200 + | > loss_text_ce: 0.041021157056093216 (0.05617901184336546) + | > loss_mel_ce: 3.5063130855560303 (3.728047030187037) + | > loss: 0.8868335485458374 (0.9460565108353962) + | > current_lr: 5e-06 + | > step_time: 0.449 (0.24513736158120827) + | > loader_time: 0.014 (0.01204537467878372) + + + --> TIME: 2024-10-20 00:06:57 -- STEP: 12250/69059 -- GLOBAL_STEP: 12250 + | > loss_text_ce: 0.0425238236784935 (0.056115398622896946) + | > loss_mel_ce: 3.3244216442108154 (3.7271206783372457) + | > loss: 0.841736376285553 (0.9458090195509841) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24508468474174033) + | > loader_time: 0.0103 (0.012042857306344159) + + + --> TIME: 2024-10-20 00:07:13 -- STEP: 12300/69059 -- GLOBAL_STEP: 12300 + | > loss_text_ce: 0.04200805723667145 (0.05605245836593972) + | > loss_mel_ce: 3.2282092571258545 (3.7262338523748375) + | > loss: 0.8175543546676636 (0.945571577989957) + | > current_lr: 5e-06 + | > step_time: 0.1566 (0.2449429965019226) + | > loader_time: 0.0115 (0.012039566776616778) + + + --> TIME: 2024-10-20 00:07:33 -- STEP: 12350/69059 -- GLOBAL_STEP: 12350 + | > loss_text_ce: 0.0430900976061821 (0.05598953328544044) + | > loss_mel_ce: 3.198247194290161 (3.7251908333195294) + | > loss: 0.810334324836731 (0.9452950919591452) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.24506118762830972) + | > loader_time: 0.0106 (0.012039605654202966) + + + --> TIME: 2024-10-20 00:07:52 -- STEP: 12400/69059 -- GLOBAL_STEP: 12400 + | > loss_text_ce: 0.04120773822069168 (0.05592874327374056) + | > loss_mel_ce: 3.3895537853240967 (3.7240816577211433) + | > loss: 0.8576903939247131 (0.9450026005602641) + | > current_lr: 5e-06 + | > step_time: 0.1517 (0.2450591177901914) + | > loader_time: 0.0102 (0.01203770116452247) + + + --> TIME: 2024-10-20 00:08:10 -- STEP: 12450/69059 -- GLOBAL_STEP: 12450 + | > loss_text_ce: 0.041170280426740646 (0.055867883079501746) + | > loss_mel_ce: 3.42431378364563 (3.7228031966772352) + | > loss: 0.8663710355758667 (0.9446677702545629) + | > current_lr: 5e-06 + | > step_time: 0.1941 (0.24502917647840508) + | > loader_time: 0.0112 (0.012035642531980943) + + + --> TIME: 2024-10-20 00:08:29 -- STEP: 12500/69059 -- GLOBAL_STEP: 12500 + | > loss_text_ce: 0.04045853018760681 (0.05580629563063358) + | > loss_mel_ce: 3.115837335586548 (3.7218742715263295) + | > loss: 0.7890739440917969 (0.9444201421022403) + | > current_lr: 5e-06 + | > step_time: 0.3467 (0.24504788820266724) + | > loader_time: 0.0157 (0.012034773693084708) + + + --> TIME: 2024-10-20 00:08:49 -- STEP: 12550/69059 -- GLOBAL_STEP: 12550 + | > loss_text_ce: 0.03879387676715851 (0.05574641751340636) + | > loss_mel_ce: 3.672250509262085 (3.7210042349370758) + | > loss: 0.9277610778808594 (0.9441876634136127) + | > current_lr: 5e-06 + | > step_time: 0.2022 (0.24513793704044295) + | > loader_time: 0.0106 (0.012032914807596995) + + + --> TIME: 2024-10-20 00:09:08 -- STEP: 12600/69059 -- GLOBAL_STEP: 12600 + | > loss_text_ce: 0.04126971960067749 (0.055685014556797474) + | > loss_mel_ce: 3.4468741416931152 (3.7201631294924047) + | > loss: 0.8720359802246094 (0.9439620363097331) + | > current_lr: 5e-06 + | > step_time: 0.1718 (0.24515429072909886) + | > loader_time: 0.0108 (0.012030714844900455) + + + --> TIME: 2024-10-20 00:09:27 -- STEP: 12650/69059 -- GLOBAL_STEP: 12650 + | > loss_text_ce: 0.04847504198551178 (0.05562434143699651) + | > loss_mel_ce: 3.7868576049804688 (3.7191400415058595) + | > loss: 0.9588331580162048 (0.9436910960419835) + | > current_lr: 5e-06 + | > step_time: 0.2582 (0.24519235335791065) + | > loader_time: 0.0117 (0.012029176926895554) + + + --> TIME: 2024-10-20 00:09:45 -- STEP: 12700/69059 -- GLOBAL_STEP: 12700 + | > loss_text_ce: 0.03867213428020477 (0.055564142109195296) + | > loss_mel_ce: 3.509922981262207 (3.718134227287105) + | > loss: 0.8871487975120544 (0.9434245926657988) + | > current_lr: 5e-06 + | > step_time: 0.1637 (0.24518750427276131) + | > loader_time: 0.0108 (0.012027525751609492) + + + --> TIME: 2024-10-20 00:10:02 -- STEP: 12750/69059 -- GLOBAL_STEP: 12750 + | > loss_text_ce: 0.038488585501909256 (0.055502217175621364) + | > loss_mel_ce: 3.155449628829956 (3.716839517929968) + | > loss: 0.7984845638275146 (0.9430854341142306) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.2451034804325478) + | > loader_time: 0.0106 (0.012025198880363904) + + + --> TIME: 2024-10-20 00:10:21 -- STEP: 12800/69059 -- GLOBAL_STEP: 12800 + | > loss_text_ce: 0.04092518612742424 (0.055442558870999824) + | > loss_mel_ce: 3.2930991649627686 (3.7158955232053925) + | > loss: 0.8335061073303223 (0.9428345208615053) + | > current_lr: 5e-06 + | > step_time: 0.1654 (0.24512326329946518) + | > loader_time: 0.0109 (0.012024328820407382) + + + --> TIME: 2024-10-20 00:10:38 -- STEP: 12850/69059 -- GLOBAL_STEP: 12850 + | > loss_text_ce: 0.03844640031456947 (0.05538260276244515) + | > loss_mel_ce: 3.346121311187744 (3.714711266194793) + | > loss: 0.8461419343948364 (0.9425234675917633) + | > current_lr: 5e-06 + | > step_time: 0.1787 (0.2450367790437394) + | > loader_time: 0.0107 (0.012020824001920817) + + + --> TIME: 2024-10-20 00:10:55 -- STEP: 12900/69059 -- GLOBAL_STEP: 12900 + | > loss_text_ce: 0.04113544896245003 (0.05532156245421969) + | > loss_mel_ce: 3.5436060428619385 (3.7136519812983066) + | > loss: 0.8961853981018066 (0.9422433862852484) + | > current_lr: 5e-06 + | > step_time: 0.1585 (0.24496672109115955) + | > loader_time: 0.0114 (0.012019046794536492) + + + --> TIME: 2024-10-20 00:11:14 -- STEP: 12950/69059 -- GLOBAL_STEP: 12950 + | > loss_text_ce: 0.039413999766111374 (0.05526402578678346) + | > loss_mel_ce: 3.7053332328796387 (3.712680456592303) + | > loss: 0.9361867904663086 (0.9419861209392536) + | > current_lr: 5e-06 + | > step_time: 0.2335 (0.2449558196564899) + | > loader_time: 0.0108 (0.012018720081874294) + + + --> TIME: 2024-10-20 00:11:33 -- STEP: 13000/69059 -- GLOBAL_STEP: 13000 + | > loss_text_ce: 0.03784407675266266 (0.055205951200368066) + | > loss_mel_ce: 3.391167163848877 (3.711709156898345) + | > loss: 0.857252836227417 (0.941728777362749) + | > current_lr: 5e-06 + | > step_time: 0.1577 (0.24495688821719244) + | > loader_time: 0.0109 (0.012016853497578538) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_13000.pth + + --> TIME: 2024-10-20 00:12:08 -- STEP: 13050/69059 -- GLOBAL_STEP: 13050 + | > loss_text_ce: 0.04664080962538719 (0.05514867204673193) + | > loss_mel_ce: 3.4897985458374023 (3.7106583529322474) + | > loss: 0.8841098546981812 (0.9414517565796647) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.24499899179085918) + | > loader_time: 0.0116 (0.012016090816921648) + + + --> TIME: 2024-10-20 00:12:28 -- STEP: 13100/69059 -- GLOBAL_STEP: 13100 + | > loss_text_ce: 0.03905384615063667 (0.05509395929607929) + | > loss_mel_ce: 3.398550510406494 (3.709696945052104) + | > loss: 0.8594011068344116 (0.9411977264171327) + | > current_lr: 5e-06 + | > step_time: 0.1562 (0.24506994500415016) + | > loader_time: 0.0106 (0.012015515957169851) + + + --> TIME: 2024-10-20 00:12:45 -- STEP: 13150/69059 -- GLOBAL_STEP: 13150 + | > loss_text_ce: 0.04117841273546219 (0.05503638956709935) + | > loss_mel_ce: 3.478585720062256 (3.708561797740335) + | > loss: 0.8799410462379456 (0.9408995471571777) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.244973892367838) + | > loader_time: 0.0109 (0.012013236343180717) + + + --> TIME: 2024-10-20 00:13:02 -- STEP: 13200/69059 -- GLOBAL_STEP: 13200 + | > loss_text_ce: 0.036848608404397964 (0.054979233843358837) + | > loss_mel_ce: 3.2469234466552734 (3.7075586402958023) + | > loss: 0.8209429979324341 (0.9406344688706315) + | > current_lr: 5e-06 + | > step_time: 0.1669 (0.24491409534757788) + | > loader_time: 0.0143 (0.012012273578932782) + + + --> TIME: 2024-10-20 00:13:20 -- STEP: 13250/69059 -- GLOBAL_STEP: 13250 + | > loss_text_ce: 0.04213152825832367 (0.054922322279439544) + | > loss_mel_ce: 3.320383310317993 (3.7064473471911383) + | > loss: 0.8406286835670471 (0.9403424176989851) + | > current_lr: 5e-06 + | > step_time: 0.2565 (0.24488689620539827) + | > loader_time: 0.0132 (0.012012414698330852) + + + --> TIME: 2024-10-20 00:13:39 -- STEP: 13300/69059 -- GLOBAL_STEP: 13300 + | > loss_text_ce: 0.03977663815021515 (0.05486546503674019) + | > loss_mel_ce: 3.0989298820495605 (3.705334680761603) + | > loss: 0.7846766114234924 (0.9400500367817115) + | > current_lr: 5e-06 + | > step_time: 0.1557 (0.2448720269095629) + | > loader_time: 0.0106 (0.012011529682274143) + + + --> TIME: 2024-10-20 00:13:57 -- STEP: 13350/69059 -- GLOBAL_STEP: 13350 + | > loss_text_ce: 0.0370476171374321 (0.05481027806742784) + | > loss_mel_ce: 3.4976377487182617 (3.7042916823862133) + | > loss: 0.8836713433265686 (0.9397754904304093) + | > current_lr: 5e-06 + | > step_time: 0.2164 (0.24486237545584918) + | > loader_time: 0.0116 (0.01201032538539014) + + + --> TIME: 2024-10-20 00:14:15 -- STEP: 13400/69059 -- GLOBAL_STEP: 13400 + | > loss_text_ce: 0.036737967282533646 (0.05475440641976313) + | > loss_mel_ce: 3.4239461421966553 (3.7034100912755927) + | > loss: 0.8651710152626038 (0.9395411247281875) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.244831328730085) + | > loader_time: 0.0105 (0.012008292390339402) + + + --> TIME: 2024-10-20 00:14:33 -- STEP: 13450/69059 -- GLOBAL_STEP: 13450 + | > loss_text_ce: 0.043397821485996246 (0.05469893090482293) + | > loss_mel_ce: 3.6171109676361084 (3.702540127903106) + | > loss: 0.9151272177696228 (0.9393097649896888) + | > current_lr: 5e-06 + | > step_time: 0.2257 (0.24481603264365498) + | > loader_time: 0.0116 (0.012006624214711234) + + + --> TIME: 2024-10-20 00:14:54 -- STEP: 13500/69059 -- GLOBAL_STEP: 13500 + | > loss_text_ce: 0.041505616158246994 (0.054645157568984325) + | > loss_mel_ce: 3.3964500427246094 (3.701776178642549) + | > loss: 0.8594889044761658 (0.9391053343349022) + | > current_lr: 5e-06 + | > step_time: 0.233 (0.24489847186759667) + | > loader_time: 0.0125 (0.012005695095768673) + + + --> TIME: 2024-10-20 00:15:12 -- STEP: 13550/69059 -- GLOBAL_STEP: 13550 + | > loss_text_ce: 0.042083363980054855 (0.054590031031202774) + | > loss_mel_ce: 3.657468795776367 (3.7008551301815347) + | > loss: 0.924888014793396 (0.9388612905683542) + | > current_lr: 5e-06 + | > step_time: 0.3179 (0.2449236536553865) + | > loader_time: 0.0122 (0.01200454134782741) + + + --> TIME: 2024-10-20 00:15:31 -- STEP: 13600/69059 -- GLOBAL_STEP: 13600 + | > loss_text_ce: 0.041750550270080566 (0.054534562585656025) + | > loss_mel_ce: 3.461782693862915 (3.6999378101264666) + | > loss: 0.8758833408355713 (0.9386180934485256) + | > current_lr: 5e-06 + | > step_time: 0.1713 (0.24494798041441862) + | > loader_time: 0.0105 (0.01200226045706692) + + + --> TIME: 2024-10-20 00:15:51 -- STEP: 13650/69059 -- GLOBAL_STEP: 13650 + | > loss_text_ce: 0.03608251363039017 (0.05447853892523049) + | > loss_mel_ce: 3.449002504348755 (3.69903342407701) + | > loss: 0.8712712526321411 (0.93837799102832) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24499445501264636) + | > loader_time: 0.0102 (0.012001230480906719) + + + --> TIME: 2024-10-20 00:16:09 -- STEP: 13700/69059 -- GLOBAL_STEP: 13700 + | > loss_text_ce: 0.042302198708057404 (0.054425663775118364) + | > loss_mel_ce: 3.3174636363983154 (3.6979315439627927) + | > loss: 0.8399414420127869 (0.9380893021995993) + | > current_lr: 5e-06 + | > step_time: 0.182 (0.2449521913667665) + | > loader_time: 0.0107 (0.011999277372429833) + + + --> TIME: 2024-10-20 00:16:28 -- STEP: 13750/69059 -- GLOBAL_STEP: 13750 + | > loss_text_ce: 0.03977575525641441 (0.0543736347431484) + | > loss_mel_ce: 3.637784481048584 (3.6971285060188923) + | > loss: 0.919390082359314 (0.9378755354491136) + | > current_lr: 5e-06 + | > step_time: 0.3368 (0.2450025472467596) + | > loader_time: 0.0145 (0.011999014056812625) + + + --> TIME: 2024-10-20 00:16:49 -- STEP: 13800/69059 -- GLOBAL_STEP: 13800 + | > loss_text_ce: 0.03874242305755615 (0.054322751106667115) + | > loss_mel_ce: 3.2197439670562744 (3.696371729650353) + | > loss: 0.8146215677261353 (0.9376736204563697) + | > current_lr: 5e-06 + | > step_time: 0.1579 (0.24511780481407608) + | > loader_time: 0.0112 (0.011997911256292584) + + + --> TIME: 2024-10-20 00:17:08 -- STEP: 13850/69059 -- GLOBAL_STEP: 13850 + | > loss_text_ce: 0.03993092477321625 (0.05427139493129075) + | > loss_mel_ce: 3.4421603679656982 (3.6956060448278096) + | > loss: 0.8705227971076965 (0.937469360212986) + | > current_lr: 5e-06 + | > step_time: 0.2229 (0.24513927726539034) + | > loader_time: 0.0112 (0.011995883797049944) + + + --> TIME: 2024-10-20 00:17:26 -- STEP: 13900/69059 -- GLOBAL_STEP: 13900 + | > loss_text_ce: 0.03570828214287758 (0.05421966459441291) + | > loss_mel_ce: 3.3998525142669678 (3.694726068716249) + | > loss: 0.858890175819397 (0.9372364335994916) + | > current_lr: 5e-06 + | > step_time: 0.1632 (0.24515087038492986) + | > loader_time: 0.011 (0.01199470152957833) + + + --> TIME: 2024-10-20 00:17:44 -- STEP: 13950/69059 -- GLOBAL_STEP: 13950 + | > loss_text_ce: 0.0422787070274353 (0.05416721509989849) + | > loss_mel_ce: 3.568385124206543 (3.693797332719229) + | > loss: 0.9026659727096558 (0.9369911372277034) + | > current_lr: 5e-06 + | > step_time: 0.5313 (0.24510042850261948) + | > loader_time: 0.0137 (0.011993186704574084) + + + --> TIME: 2024-10-20 00:18:02 -- STEP: 14000/69059 -- GLOBAL_STEP: 14000 + | > loss_text_ce: 0.04039999842643738 (0.05411504297304343) + | > loss_mel_ce: 3.4334394931793213 (3.692891673513815) + | > loss: 0.8684598803520203 (0.9367516793991828) + | > current_lr: 5e-06 + | > step_time: 0.1907 (0.24507268021787915) + | > loader_time: 0.0106 (0.011991255879402153) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_14000.pth + + --> TIME: 2024-10-20 00:18:36 -- STEP: 14050/69059 -- GLOBAL_STEP: 14050 + | > loss_text_ce: 0.041387561708688736 (0.0540612520911195) + | > loss_mel_ce: 3.5849075317382812 (3.6919724255015356) + | > loss: 0.9065737724304199 (0.936508419666425) + | > current_lr: 5e-06 + | > step_time: 0.406 (0.24508802923019246) + | > loader_time: 0.0171 (0.011991053476028163) + + + --> TIME: 2024-10-20 00:18:54 -- STEP: 14100/69059 -- GLOBAL_STEP: 14100 + | > loss_text_ce: 0.038784001022577286 (0.05400878138073981) + | > loss_mel_ce: 3.270218849182129 (3.6910284939556197) + | > loss: 0.8272507190704346 (0.9362593190889822) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.24504672789404577) + | > loader_time: 0.0114 (0.011990717810096462) + + + --> TIME: 2024-10-20 00:19:13 -- STEP: 14150/69059 -- GLOBAL_STEP: 14150 + | > loss_text_ce: 0.04258181154727936 (0.05395693809715449) + | > loss_mel_ce: 3.8078806400299072 (3.690221539773699) + | > loss: 0.9626156091690063 (0.9360446197236796) + | > current_lr: 5e-06 + | > step_time: 0.5069 (0.24508090002376703) + | > loader_time: 0.0167 (0.011990819802975056) + + + --> TIME: 2024-10-20 00:19:31 -- STEP: 14200/69059 -- GLOBAL_STEP: 14200 + | > loss_text_ce: 0.03897981718182564 (0.05390340396058789) + | > loss_mel_ce: 3.756791353225708 (3.6892097907334924) + | > loss: 0.9489427804946899 (0.9357782989362585) + | > current_lr: 5e-06 + | > step_time: 0.4972 (0.2450443035280201) + | > loader_time: 0.015 (0.0119899274933506) + + + --> TIME: 2024-10-20 00:19:49 -- STEP: 14250/69059 -- GLOBAL_STEP: 14250 + | > loss_text_ce: 0.03772788122296333 (0.05385200102627255) + | > loss_mel_ce: 3.1639163494110107 (3.688267109017617) + | > loss: 0.8004110455513 (0.935529777765273) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.24500009151927213) + | > loader_time: 0.0116 (0.011988846092893359) + + + --> TIME: 2024-10-20 00:20:07 -- STEP: 14300/69059 -- GLOBAL_STEP: 14300 + | > loss_text_ce: 0.03750402107834816 (0.05380312580693413) + | > loss_mel_ce: 3.1338069438934326 (3.687471893400573) + | > loss: 0.7928277254104614 (0.9353187550614753) + | > current_lr: 5e-06 + | > step_time: 0.1642 (0.2450125777804768) + | > loader_time: 0.0108 (0.011989102113497) + + + --> TIME: 2024-10-20 00:20:27 -- STEP: 14350/69059 -- GLOBAL_STEP: 14350 + | > loss_text_ce: 0.037248317152261734 (0.05375376388145011) + | > loss_mel_ce: 3.2395055294036865 (3.6866977273960915) + | > loss: 0.8191884756088257 (0.9351128730748994) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.2450814386693443) + | > loader_time: 0.012 (0.011989458745364936) + + + --> TIME: 2024-10-20 00:20:45 -- STEP: 14400/69059 -- GLOBAL_STEP: 14400 + | > loss_text_ce: 0.03835292160511017 (0.05370552368565544) + | > loss_mel_ce: 3.432990550994873 (3.685986733883613) + | > loss: 0.8678358793258667 (0.9349230646424813) + | > current_lr: 5e-06 + | > step_time: 0.1639 (0.2450707516239749) + | > loader_time: 0.0115 (0.011987725496292106) + + + --> TIME: 2024-10-20 00:21:03 -- STEP: 14450/69059 -- GLOBAL_STEP: 14450 + | > loss_text_ce: 0.0340985544025898 (0.05365578129861778) + | > loss_mel_ce: 3.392867088317871 (3.685025286591996) + | > loss: 0.8567414283752441 (0.9346702672205984) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.24503692028019255) + | > loader_time: 0.0111 (0.01198647528783672) + + + --> TIME: 2024-10-20 00:21:22 -- STEP: 14500/69059 -- GLOBAL_STEP: 14500 + | > loss_text_ce: 0.03599928319454193 (0.053606421719850944) + | > loss_mel_ce: 3.4772422313690186 (3.6841733911777306) + | > loss: 0.8783103823661804 (0.9344449534662829) + | > current_lr: 5e-06 + | > step_time: 0.3769 (0.24503470057454602) + | > loader_time: 0.0138 (0.011993543707091225) + + + --> TIME: 2024-10-20 00:21:40 -- STEP: 14550/69059 -- GLOBAL_STEP: 14550 + | > loss_text_ce: 0.038616444915533066 (0.05355557951372082) + | > loss_mel_ce: 3.8414852619171143 (3.6832683447382233) + | > loss: 0.9700254201889038 (0.9342059813045545) + | > current_lr: 5e-06 + | > step_time: 0.3375 (0.2450273619320794) + | > loader_time: 0.0121 (0.011991957979103946) + + + --> TIME: 2024-10-20 00:21:59 -- STEP: 14600/69059 -- GLOBAL_STEP: 14600 + | > loss_text_ce: 0.03607259318232536 (0.053507577958070034) + | > loss_mel_ce: 3.1089606285095215 (3.6824151914087) + | > loss: 0.7862582802772522 (0.9339806925760546) + | > current_lr: 5e-06 + | > step_time: 0.1515 (0.24501810289409062) + | > loader_time: 0.0105 (0.011991100033668614) + + + --> TIME: 2024-10-20 00:22:18 -- STEP: 14650/69059 -- GLOBAL_STEP: 14650 + | > loss_text_ce: 0.04035351797938347 (0.05346068894369792) + | > loss_mel_ce: 3.126638412475586 (3.6815009779816137) + | > loss: 0.7917479872703552 (0.9337404169564351) + | > current_lr: 5e-06 + | > step_time: 0.1959 (0.24504992670573472) + | > loader_time: 0.0113 (0.011990401102007445) + + + --> TIME: 2024-10-20 00:22:35 -- STEP: 14700/69059 -- GLOBAL_STEP: 14700 + | > loss_text_ce: 0.03780205547809601 (0.053410921321839486) + | > loss_mel_ce: 3.6109211444854736 (3.6805193900575386) + | > loss: 0.9121807813644409 (0.9334825780724172) + | > current_lr: 5e-06 + | > step_time: 0.3071 (0.244961655172361) + | > loader_time: 0.0124 (0.011987737847023262) + + + --> TIME: 2024-10-20 00:22:55 -- STEP: 14750/69059 -- GLOBAL_STEP: 14750 + | > loss_text_ce: 0.040486838668584824 (0.05336361659558121) + | > loss_mel_ce: 3.418348789215088 (3.679811883651598) + | > loss: 0.8647089004516602 (0.9332938753022976) + | > current_lr: 5e-06 + | > step_time: 0.2047 (0.24505185990414377) + | > loader_time: 0.0106 (0.011987306417044938) + + + --> TIME: 2024-10-20 00:23:14 -- STEP: 14800/69059 -- GLOBAL_STEP: 14800 + | > loss_text_ce: 0.03554718568921089 (0.05331699238939038) + | > loss_mel_ce: 3.389925479888916 (3.6789055115951017) + | > loss: 0.8563681840896606 (0.9330556262344918) + | > current_lr: 5e-06 + | > step_time: 0.1962 (0.24505780336019156) + | > loader_time: 0.0115 (0.01198543242506078) + + + --> TIME: 2024-10-20 00:23:31 -- STEP: 14850/69059 -- GLOBAL_STEP: 14850 + | > loss_text_ce: 0.038802895694971085 (0.05327049232291833) + | > loss_mel_ce: 3.2766776084899902 (3.677895731075037) + | > loss: 0.8288701176643372 (0.9327915560837938) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.24499273444667008) + | > loader_time: 0.0107 (0.01198313684174508) + + + --> TIME: 2024-10-20 00:23:48 -- STEP: 14900/69059 -- GLOBAL_STEP: 14900 + | > loss_text_ce: 0.04076066240668297 (0.053223574098024376) + | > loss_mel_ce: 3.6013612747192383 (3.677048045436802) + | > loss: 0.9105305075645447 (0.9325679051316015) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.24490482939969774) + | > loader_time: 0.0106 (0.011979815047859338) + + + --> TIME: 2024-10-20 00:24:05 -- STEP: 14950/69059 -- GLOBAL_STEP: 14950 + | > loss_text_ce: 0.034636542201042175 (0.05317611575151545) + | > loss_mel_ce: 3.304534435272217 (3.6760348726674406) + | > loss: 0.8347927331924438 (0.9323027473428965) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.2448410148524919) + | > loader_time: 0.0104 (0.011977022483596028) + + + --> TIME: 2024-10-20 00:24:23 -- STEP: 15000/69059 -- GLOBAL_STEP: 15000 + | > loss_text_ce: 0.04212935268878937 (0.053129277543971724) + | > loss_mel_ce: 3.3553595542907715 (3.674979454437886) + | > loss: 0.8493722081184387 (0.9320271832267434) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.2448332683245341) + | > loader_time: 0.0107 (0.01197480670611063) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_15000.pth + + --> TIME: 2024-10-20 00:24:55 -- STEP: 15050/69059 -- GLOBAL_STEP: 15050 + | > loss_text_ce: 0.03949689492583275 (0.05308156247427078) + | > loss_mel_ce: 3.358886957168579 (3.67408611828306) + | > loss: 0.8495959639549255 (0.9317919204163779) + | > current_lr: 5e-06 + | > step_time: 0.2054 (0.24480132771488836) + | > loader_time: 0.0109 (0.011974939983152464) + + + --> TIME: 2024-10-20 00:25:14 -- STEP: 15100/69059 -- GLOBAL_STEP: 15100 + | > loss_text_ce: 0.04141734540462494 (0.05303779344808363) + | > loss_mel_ce: 3.7934682369232178 (3.6731485085771514) + | > loss: 0.958721399307251 (0.9315465757349459) + | > current_lr: 5e-06 + | > step_time: 0.4524 (0.2448228282486366) + | > loader_time: 0.0192 (0.011974894984668446) + + + --> TIME: 2024-10-20 00:25:34 -- STEP: 15150/69059 -- GLOBAL_STEP: 15150 + | > loss_text_ce: 0.03662395849823952 (0.052992424452707895) + | > loss_mel_ce: 3.6340630054473877 (3.67242130029319) + | > loss: 0.9176717400550842 (0.9313534314207502) + | > current_lr: 5e-06 + | > step_time: 0.2061 (0.2449001994463477) + | > loader_time: 0.0113 (0.011977316286697633) + + + --> TIME: 2024-10-20 00:25:53 -- STEP: 15200/69059 -- GLOBAL_STEP: 15200 + | > loss_text_ce: 0.0398826077580452 (0.05294642881980439) + | > loss_mel_ce: 3.5170366764068604 (3.6716264946993973) + | > loss: 0.8892298340797424 (0.9311432311134894) + | > current_lr: 5e-06 + | > step_time: 0.3123 (0.24490845495148708) + | > loader_time: 0.0142 (0.011977434221066919) + + + --> TIME: 2024-10-20 00:26:11 -- STEP: 15250/69059 -- GLOBAL_STEP: 15250 + | > loss_text_ce: 0.0382113941013813 (0.05290203094042695) + | > loss_mel_ce: 3.3474879264831543 (3.6707859429375014) + | > loss: 0.8464248180389404 (0.9309219937050919) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.2448906557364542) + | > loader_time: 0.0118 (0.011977001659205695) + + + --> TIME: 2024-10-20 00:26:29 -- STEP: 15300/69059 -- GLOBAL_STEP: 15300 + | > loss_text_ce: 0.03691377490758896 (0.05285486820184307) + | > loss_mel_ce: 3.2792422771453857 (3.6700193908946908) + | > loss: 0.8290390372276306 (0.9307185650066603) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24486379191766378) + | > loader_time: 0.0113 (0.011975991445429178) + + + --> TIME: 2024-10-20 00:26:47 -- STEP: 15350/69059 -- GLOBAL_STEP: 15350 + | > loss_text_ce: 0.04195340722799301 (0.052808568237124695) + | > loss_mel_ce: 3.5038630962371826 (3.6691679823282044) + | > loss: 0.8864541053771973 (0.9304941378587223) + | > current_lr: 5e-06 + | > step_time: 0.2623 (0.24481533659397586) + | > loader_time: 0.0117 (0.011973761176441694) + + + --> TIME: 2024-10-20 00:27:06 -- STEP: 15400/69059 -- GLOBAL_STEP: 15400 + | > loss_text_ce: 0.03832762688398361 (0.052765239490461996) + | > loss_mel_ce: 3.255767345428467 (3.668309330491271) + | > loss: 0.823523759841919 (0.9302686427127221) + | > current_lr: 5e-06 + | > step_time: 0.1567 (0.24485083781279526) + | > loader_time: 0.0105 (0.011973837273461472) + + + --> TIME: 2024-10-20 00:27:26 -- STEP: 15450/69059 -- GLOBAL_STEP: 15450 + | > loss_text_ce: 0.040071699768304825 (0.05272191286713925) + | > loss_mel_ce: 3.515575885772705 (3.6675012886022618) + | > loss: 0.8889119029045105 (0.9300558005876131) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24489319994225858) + | > loader_time: 0.011 (0.011973926704678327) + + + --> TIME: 2024-10-20 00:27:44 -- STEP: 15500/69059 -- GLOBAL_STEP: 15500 + | > loss_text_ce: 0.0369856171309948 (0.052677197534951266) + | > loss_mel_ce: 3.334557056427002 (3.6668094854970077) + | > loss: 0.8428856730461121 (0.9298716709844518) + | > current_lr: 5e-06 + | > step_time: 0.1652 (0.24487517902928013) + | > loader_time: 0.0107 (0.01197233263138801) + + + --> TIME: 2024-10-20 00:28:01 -- STEP: 15550/69059 -- GLOBAL_STEP: 15550 + | > loss_text_ce: 0.04129119962453842 (0.05263363990489481) + | > loss_mel_ce: 3.2245261669158936 (3.665748418489064) + | > loss: 0.8164543509483337 (0.9295955148195522) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.24479580575630214) + | > loader_time: 0.0107 (0.011970182875728293) + + + --> TIME: 2024-10-20 00:28:20 -- STEP: 15600/69059 -- GLOBAL_STEP: 15600 + | > loss_text_ce: 0.03932873159646988 (0.052588702219848626) + | > loss_mel_ce: 3.6000816822052 (3.6649894184485405) + | > loss: 0.9098526239395142 (0.9293945303979583) + | > current_lr: 5e-06 + | > step_time: 0.2978 (0.2448467926795666) + | > loader_time: 0.012 (0.011969427955456262) + + + --> TIME: 2024-10-20 00:28:39 -- STEP: 15650/69059 -- GLOBAL_STEP: 15650 + | > loss_text_ce: 0.043289653956890106 (0.05254767342449738) + | > loss_mel_ce: 3.607279062271118 (3.664147150356543) + | > loss: 0.9126421809196472 (0.929173706168183) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.24484922409057616) + | > loader_time: 0.0106 (0.011969083993198765) + + + --> TIME: 2024-10-20 00:28:57 -- STEP: 15700/69059 -- GLOBAL_STEP: 15700 + | > loss_text_ce: 0.03539826720952988 (0.052503768070155825) + | > loss_mel_ce: 3.4774818420410156 (3.663379559835805) + | > loss: 0.8782200217247009 (0.9289708321982879) + | > current_lr: 5e-06 + | > step_time: 0.1894 (0.24483582211148208) + | > loader_time: 0.011 (0.011967860726034557) + + + --> TIME: 2024-10-20 00:29:16 -- STEP: 15750/69059 -- GLOBAL_STEP: 15750 + | > loss_text_ce: 0.04021966829895973 (0.05246009922784452) + | > loss_mel_ce: 3.0175862312316895 (3.662509448248237) + | > loss: 0.7644515037536621 (0.9287423870904096) + | > current_lr: 5e-06 + | > step_time: 0.2056 (0.2448447219303676) + | > loader_time: 0.0113 (0.011966776984078536) + + + --> TIME: 2024-10-20 00:29:34 -- STEP: 15800/69059 -- GLOBAL_STEP: 15800 + | > loss_text_ce: 0.03889734297990799 (0.052415825645615956) + | > loss_mel_ce: 3.4727303981781006 (3.6616362964654217) + | > loss: 0.8779069185256958 (0.9285130307342424) + | > current_lr: 5e-06 + | > step_time: 0.2268 (0.24485396151301228) + | > loader_time: 0.0124 (0.011966284287126752) + + + --> TIME: 2024-10-20 00:29:52 -- STEP: 15850/69059 -- GLOBAL_STEP: 15850 + | > loss_text_ce: 0.041249699890613556 (0.05237125676466067) + | > loss_mel_ce: 3.661088228225708 (3.6607877569469314) + | > loss: 0.9255844950675964 (0.9282897536386066) + | > current_lr: 5e-06 + | > step_time: 0.3415 (0.24479769587892838) + | > loader_time: 0.0127 (0.01196489446922807) + + + --> TIME: 2024-10-20 00:30:11 -- STEP: 15900/69059 -- GLOBAL_STEP: 15900 + | > loss_text_ce: 0.03932931274175644 (0.05232945082580311) + | > loss_mel_ce: 3.2543210983276367 (3.6599441535967645) + | > loss: 0.8234125971794128 (0.9280684013261726) + | > current_lr: 5e-06 + | > step_time: 0.1973 (0.24482156657572812) + | > loader_time: 0.0108 (0.011964332982429162) + + + --> TIME: 2024-10-20 00:30:29 -- STEP: 15950/69059 -- GLOBAL_STEP: 15950 + | > loss_text_ce: 0.03818952664732933 (0.05228839091699679) + | > loss_mel_ce: 3.371473789215088 (3.659182763772319) + | > loss: 0.8524158000946045 (0.927867788887322) + | > current_lr: 5e-06 + | > step_time: 0.256 (0.24480155309539603) + | > loader_time: 0.0117 (0.011962660113845867) + + + --> TIME: 2024-10-20 00:30:47 -- STEP: 16000/69059 -- GLOBAL_STEP: 16000 + | > loss_text_ce: 0.037912189960479736 (0.05224599523609484) + | > loss_mel_ce: 3.227754592895508 (3.6583017049580757) + | > loss: 0.8164166808128357 (0.927636925276368) + | > current_lr: 5e-06 + | > step_time: 0.1573 (0.24478084681928158) + | > loader_time: 0.0101 (0.011961185410618774) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_16000.pth + + --> TIME: 2024-10-20 00:31:18 -- STEP: 16050/69059 -- GLOBAL_STEP: 16050 + | > loss_text_ce: 0.035045478492975235 (0.05220319515865891) + | > loss_mel_ce: 3.511678457260132 (3.6573951800515663) + | > loss: 0.8866809606552124 (0.927399594029904) + | > current_lr: 5e-06 + | > step_time: 0.2122 (0.2447394260364901) + | > loader_time: 0.0112 (0.011960910844654301) + + + --> TIME: 2024-10-20 00:31:36 -- STEP: 16100/69059 -- GLOBAL_STEP: 16100 + | > loss_text_ce: 0.03651722893118858 (0.05216039384767301) + | > loss_mel_ce: 3.6492056846618652 (3.656733278147177) + | > loss: 0.9214307069778442 (0.9272234182328162) + | > current_lr: 5e-06 + | > step_time: 0.1583 (0.244683749779411) + | > loader_time: 0.0121 (0.011960641330813763) + + + --> TIME: 2024-10-20 00:31:54 -- STEP: 16150/69059 -- GLOBAL_STEP: 16150 + | > loss_text_ce: 0.03925715759396553 (0.052117330073249035) + | > loss_mel_ce: 3.5097367763519287 (3.6560309077047286) + | > loss: 0.8872484564781189 (0.9270370596887149) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.24467823199812472) + | > loader_time: 0.0114 (0.011960483187861479) + + + --> TIME: 2024-10-20 00:32:11 -- STEP: 16200/69059 -- GLOBAL_STEP: 16200 + | > loss_text_ce: 0.03850893676280975 (0.0520752052774214) + | > loss_mel_ce: 3.1884989738464355 (3.655283003074146) + | > loss: 0.8067519664764404 (0.9268395523303812) + | > current_lr: 5e-06 + | > step_time: 0.1575 (0.24459240318816386) + | > loader_time: 0.0107 (0.011959573766331607) + + + --> TIME: 2024-10-20 00:32:30 -- STEP: 16250/69059 -- GLOBAL_STEP: 16250 + | > loss_text_ce: 0.041622042655944824 (0.05203501836726279) + | > loss_mel_ce: 3.348301649093628 (3.6545196873298003) + | > loss: 0.8474808931350708 (0.9266386766543746) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24460535037700948) + | > loader_time: 0.0106 (0.011959357452392572) + + + --> TIME: 2024-10-20 00:32:48 -- STEP: 16300/69059 -- GLOBAL_STEP: 16300 + | > loss_text_ce: 0.03638195991516113 (0.051994019074399586) + | > loss_mel_ce: 3.1725082397460938 (3.653666632350963) + | > loss: 0.8022225499153137 (0.9264151630774589) + | > current_lr: 5e-06 + | > step_time: 0.154 (0.24458286696416467) + | > loader_time: 0.013 (0.011959076761468049) + + + --> TIME: 2024-10-20 00:33:07 -- STEP: 16350/69059 -- GLOBAL_STEP: 16350 + | > loss_text_ce: 0.03695063665509224 (0.051952669154795475) + | > loss_mel_ce: 3.315690517425537 (3.6528739958800984) + | > loss: 0.8381602764129639 (0.9262066664681149) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24460889313199105) + | > loader_time: 0.0104 (0.011958152351029415) + + + --> TIME: 2024-10-20 00:33:25 -- STEP: 16400/69059 -- GLOBAL_STEP: 16400 + | > loss_text_ce: 0.0388614758849144 (0.051911918762389646) + | > loss_mel_ce: 3.3864808082580566 (3.6521570523337568) + | > loss: 0.8563355803489685 (0.9260172429898881) + | > current_lr: 5e-06 + | > step_time: 0.1814 (0.2445769910550699) + | > loader_time: 0.0108 (0.011956243093420813) + + + --> TIME: 2024-10-20 00:33:44 -- STEP: 16450/69059 -- GLOBAL_STEP: 16450 + | > loss_text_ce: 0.03846581280231476 (0.051871395411824554) + | > loss_mel_ce: 3.74310302734375 (3.6513883044437017) + | > loss: 0.9453921914100647 (0.9258149251821909) + | > current_lr: 5e-06 + | > step_time: 0.5105 (0.24459791270432862) + | > loader_time: 0.014 (0.011956436090556315) + + + --> TIME: 2024-10-20 00:34:01 -- STEP: 16500/69059 -- GLOBAL_STEP: 16500 + | > loss_text_ce: 0.03729282692074776 (0.05182943320951659) + | > loss_mel_ce: 3.296372413635254 (3.650575521743653) + | > loss: 0.8334162831306458 (0.9256012389515378) + | > current_lr: 5e-06 + | > step_time: 0.1642 (0.24452761515704066) + | > loader_time: 0.0103 (0.011954778685714252) + + + --> TIME: 2024-10-20 00:34:19 -- STEP: 16550/69059 -- GLOBAL_STEP: 16550 + | > loss_text_ce: 0.04044579714536667 (0.051789675271222774) + | > loss_mel_ce: 3.174313545227051 (3.6497199661854105) + | > loss: 0.8036898374557495 (0.9253774105818242) + | > current_lr: 5e-06 + | > step_time: 0.178 (0.24451670204404616) + | > loader_time: 0.0104 (0.011953146191161917) + + + --> TIME: 2024-10-20 00:34:38 -- STEP: 16600/69059 -- GLOBAL_STEP: 16600 + | > loss_text_ce: 0.037474680691957474 (0.051749758309942585) + | > loss_mel_ce: 3.186579465866089 (3.6488196136577953) + | > loss: 0.8060135245323181 (0.9251423432094495) + | > current_lr: 5e-06 + | > step_time: 0.1528 (0.24452526312276537) + | > loader_time: 0.0136 (0.011952809396996547) + + + --> TIME: 2024-10-20 00:34:56 -- STEP: 16650/69059 -- GLOBAL_STEP: 16650 + | > loss_text_ce: 0.03806918114423752 (0.051710648162362975) + | > loss_mel_ce: 3.3847742080688477 (3.647994418344691) + | > loss: 0.8557108640670776 (0.9249262668456391) + | > current_lr: 5e-06 + | > step_time: 0.1772 (0.24449765171016652) + | > loader_time: 0.0106 (0.011951164096683346) + + + --> TIME: 2024-10-20 00:35:13 -- STEP: 16700/69059 -- GLOBAL_STEP: 16700 + | > loss_text_ce: 0.04023095220327377 (0.05167043968120536) + | > loss_mel_ce: 3.513488292694092 (3.647277762218857) + | > loss: 0.8884298205375671 (0.9247370506867666) + | > current_lr: 5e-06 + | > step_time: 0.1651 (0.24445165185871234) + | > loader_time: 0.0107 (0.011949160213242027) + + + --> TIME: 2024-10-20 00:35:32 -- STEP: 16750/69059 -- GLOBAL_STEP: 16750 + | > loss_text_ce: 0.04163108766078949 (0.051630211377766404) + | > loss_mel_ce: 3.1277849674224854 (3.64649353662177) + | > loss: 0.7923539876937866 (0.9245309372161741) + | > current_lr: 5e-06 + | > step_time: 0.2315 (0.24449221249480743) + | > loader_time: 0.0117 (0.011948340287849074) + + + --> TIME: 2024-10-20 00:35:50 -- STEP: 16800/69059 -- GLOBAL_STEP: 16800 + | > loss_text_ce: 0.038297150284051895 (0.05159134148509179) + | > loss_mel_ce: 3.517193078994751 (3.6456476601248626) + | > loss: 0.8888725638389587 (0.9243097506179683) + | > current_lr: 5e-06 + | > step_time: 0.2305 (0.24445197074186234) + | > loader_time: 0.0112 (0.011946573087147293) + + + --> TIME: 2024-10-20 00:36:08 -- STEP: 16850/69059 -- GLOBAL_STEP: 16850 + | > loss_text_ce: 0.038146909326314926 (0.05155267429148392) + | > loss_mel_ce: 3.478677272796631 (3.6449854589711244) + | > loss: 0.8792060613632202 (0.9241345335291107) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.24441775750689407) + | > loader_time: 0.0105 (0.011944771690255445) + + + --> TIME: 2024-10-20 00:36:26 -- STEP: 16900/69059 -- GLOBAL_STEP: 16900 + | > loss_text_ce: 0.0350656732916832 (0.05151365157503332) + | > loss_mel_ce: 3.4049367904663086 (3.644207983327329) + | > loss: 0.8600006103515625 (0.9239304089546192) + | > current_lr: 5e-06 + | > step_time: 0.1563 (0.24440829799020078) + | > loader_time: 0.011 (0.011943186599121986) + + + --> TIME: 2024-10-20 00:36:45 -- STEP: 16950/69059 -- GLOBAL_STEP: 16950 + | > loss_text_ce: 0.040270403027534485 (0.051476872004238734) + | > loss_mel_ce: 3.2088167667388916 (3.643450792743036) + | > loss: 0.812271773815155 (0.923731916407919) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.2444402222084788) + | > loader_time: 0.0103 (0.011941889295887447) + + + --> TIME: 2024-10-20 00:37:03 -- STEP: 17000/69059 -- GLOBAL_STEP: 17000 + | > loss_text_ce: 0.03521495684981346 (0.051437427045448576) + | > loss_mel_ce: 3.4691312313079834 (3.6426086600387744) + | > loss: 0.8760865330696106 (0.9235115219915606) + | > current_lr: 5e-06 + | > step_time: 0.1975 (0.24442331238353954) + | > loader_time: 0.0106 (0.011939493445789108) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_17000.pth + + --> TIME: 2024-10-20 00:37:36 -- STEP: 17050/69059 -- GLOBAL_STEP: 17050 + | > loss_text_ce: 0.03668632730841637 (0.0513999223783392) + | > loss_mel_ce: 3.375591278076172 (3.64186096308867) + | > loss: 0.8530694246292114 (0.9233152215886318) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.2444344258518163) + | > loader_time: 0.011 (0.011938808348871035) + + + --> TIME: 2024-10-20 00:37:53 -- STEP: 17100/69059 -- GLOBAL_STEP: 17100 + | > loss_text_ce: 0.03933127224445343 (0.051361622586846185) + | > loss_mel_ce: 3.4979794025421143 (3.6410632289919875) + | > loss: 0.8843276500701904 (0.923106213114414) + | > current_lr: 5e-06 + | > step_time: 0.3583 (0.24437861208330122) + | > loader_time: 0.0131 (0.011938454073074959) + + + --> TIME: 2024-10-20 00:38:12 -- STEP: 17150/69059 -- GLOBAL_STEP: 17150 + | > loss_text_ce: 0.038918349891901016 (0.051323172053425195) + | > loss_mel_ce: 3.4987783432006836 (3.6402726860018517) + | > loss: 0.8844241499900818 (0.9228989647394014) + | > current_lr: 5e-06 + | > step_time: 0.4504 (0.244364342745122) + | > loader_time: 0.0136 (0.011937575145643578) + + + --> TIME: 2024-10-20 00:38:30 -- STEP: 17200/69059 -- GLOBAL_STEP: 17200 + | > loss_text_ce: 0.03452707827091217 (0.05128501167932376) + | > loss_mel_ce: 3.1177492141723633 (3.639538855663559) + | > loss: 0.7880690693855286 (0.9227059670693649) + | > current_lr: 5e-06 + | > step_time: 0.1646 (0.2443700365826141) + | > loader_time: 0.0104 (0.011936452319455695) + + + --> TIME: 2024-10-20 00:38:50 -- STEP: 17250/69059 -- GLOBAL_STEP: 17250 + | > loss_text_ce: 0.03473048657178879 (0.051247087035057995) + | > loss_mel_ce: 2.9833898544311523 (3.638920309978975) + | > loss: 0.7545300722122192 (0.9225418494991632) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.2444218409579733) + | > loader_time: 0.0107 (0.011936250368754066) + + + --> TIME: 2024-10-20 00:39:08 -- STEP: 17300/69059 -- GLOBAL_STEP: 17300 + | > loss_text_ce: 0.03499482944607735 (0.051210821355999685) + | > loss_mel_ce: 2.9566848278045654 (3.6381982327747813) + | > loss: 0.7479199171066284 (0.9223522637896454) + | > current_lr: 5e-06 + | > step_time: 0.1518 (0.24438728659139203) + | > loader_time: 0.0111 (0.011934811173146858) + + + --> TIME: 2024-10-20 00:39:27 -- STEP: 17350/69059 -- GLOBAL_STEP: 17350 + | > loss_text_ce: 0.0376434251666069 (0.05117502665356511) + | > loss_mel_ce: 3.369205951690674 (3.637638908312017) + | > loss: 0.8517123460769653 (0.9222034839972292) + | > current_lr: 5e-06 + | > step_time: 0.2236 (0.24442131912330387) + | > loader_time: 0.0113 (0.011934491358160628) + + + --> TIME: 2024-10-20 00:39:46 -- STEP: 17400/69059 -- GLOBAL_STEP: 17400 + | > loss_text_ce: 0.03825605660676956 (0.05113626562080317) + | > loss_mel_ce: 3.487733840942383 (3.636913816983663) + | > loss: 0.8814975023269653 (0.9220125209086237) + | > current_lr: 5e-06 + | > step_time: 0.1984 (0.24443548134003562) + | > loader_time: 0.0109 (0.011933673784650599) + + + --> TIME: 2024-10-20 00:40:03 -- STEP: 17450/69059 -- GLOBAL_STEP: 17450 + | > loss_text_ce: 0.040135953575372696 (0.05109926426884591) + | > loss_mel_ce: 3.5662076473236084 (3.6362530529874686) + | > loss: 0.9015858769416809 (0.9218380795645502) + | > current_lr: 5e-06 + | > step_time: 0.1718 (0.24439393784050273) + | > loader_time: 0.0105 (0.011932162213803695) + + + --> TIME: 2024-10-20 00:40:19 -- STEP: 17500/69059 -- GLOBAL_STEP: 17500 + | > loss_text_ce: 0.03664762154221535 (0.05106076726359963) + | > loss_mel_ce: 3.44545316696167 (3.6354133641787842) + | > loss: 0.8705251812934875 (0.9216185331038061) + | > current_lr: 5e-06 + | > step_time: 0.1641 (0.2442867220061166) + | > loader_time: 0.0104 (0.011929059587206149) + + + --> TIME: 2024-10-20 00:40:37 -- STEP: 17550/69059 -- GLOBAL_STEP: 17550 + | > loss_text_ce: 0.03377503156661987 (0.05102179005381202) + | > loss_mel_ce: 3.32303524017334 (3.63463751112293) + | > loss: 0.8392025828361511 (0.9214148255325104) + | > current_lr: 5e-06 + | > step_time: 0.2151 (0.2442578971351993) + | > loader_time: 0.0109 (0.011926977518956538) + + + --> TIME: 2024-10-20 00:40:56 -- STEP: 17600/69059 -- GLOBAL_STEP: 17600 + | > loss_text_ce: 0.03916320949792862 (0.05098480652425084) + | > loss_mel_ce: 3.47243595123291 (3.633872978619545) + | > loss: 0.8778997659683228 (0.9212144465338092) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.24426318617029624) + | > loader_time: 0.0105 (0.01192576891996643) + + + --> TIME: 2024-10-20 00:41:14 -- STEP: 17650/69059 -- GLOBAL_STEP: 17650 + | > loss_text_ce: 0.03724629431962967 (0.050948523360676366) + | > loss_mel_ce: 3.1117618083953857 (3.6330991340156222) + | > loss: 0.7872520089149475 (0.9210119145821573) + | > current_lr: 5e-06 + | > step_time: 0.1851 (0.24424651847007253) + | > loader_time: 0.0112 (0.011924706275334767) + + + --> TIME: 2024-10-20 00:41:32 -- STEP: 17700/69059 -- GLOBAL_STEP: 17700 + | > loss_text_ce: 0.04119438678026199 (0.050913269124125186) + | > loss_mel_ce: 3.203798294067383 (3.6323305781143445) + | > loss: 0.8112481832504272 (0.9208109620567086) + | > current_lr: 5e-06 + | > step_time: 0.1649 (0.2442436027796255) + | > loader_time: 0.0101 (0.01192285567353673) + + + --> TIME: 2024-10-20 00:41:51 -- STEP: 17750/69059 -- GLOBAL_STEP: 17750 + | > loss_text_ce: 0.03730976581573486 (0.05087797355588876) + | > loss_mel_ce: 3.5231270790100098 (3.6317377298516194) + | > loss: 0.8901091814041138 (0.9206539260938124) + | > current_lr: 5e-06 + | > step_time: 0.1865 (0.24425955261982663) + | > loader_time: 0.0114 (0.011922436351507469) + + + --> TIME: 2024-10-20 00:42:09 -- STEP: 17800/69059 -- GLOBAL_STEP: 17800 + | > loss_text_ce: 0.041547466069459915 (0.05084284074730067) + | > loss_mel_ce: 3.1922764778137207 (3.6308567656024024) + | > loss: 0.8084560036659241 (0.920424901836373) + | > current_lr: 5e-06 + | > step_time: 0.1647 (0.24424309899297994) + | > loader_time: 0.0105 (0.01192193688971271) + + + --> TIME: 2024-10-20 00:42:28 -- STEP: 17850/69059 -- GLOBAL_STEP: 17850 + | > loss_text_ce: 0.03995361179113388 (0.05080845511781886) + | > loss_mel_ce: 3.115907669067383 (3.6301888862310587) + | > loss: 0.7889653444290161 (0.920249335582182) + | > current_lr: 5e-06 + | > step_time: 0.3111 (0.24425941643594692) + | > loader_time: 0.0121 (0.011920678047906764) + + + --> TIME: 2024-10-20 00:42:46 -- STEP: 17900/69059 -- GLOBAL_STEP: 17900 + | > loss_text_ce: 0.037851057946681976 (0.050772798868239426) + | > loss_mel_ce: 3.1603541374206543 (3.629442250608722) + | > loss: 0.7995513081550598 (0.9200537626270467) + | > current_lr: 5e-06 + | > step_time: 0.3069 (0.24423513421799217) + | > loader_time: 0.0124 (0.011919159502956429) + + + --> TIME: 2024-10-20 00:43:04 -- STEP: 17950/69059 -- GLOBAL_STEP: 17950 + | > loss_text_ce: 0.03691687434911728 (0.050737456005263616) + | > loss_mel_ce: 3.648343801498413 (3.628566197552048) + | > loss: 0.9213151931762695 (0.9198259136437698) + | > current_lr: 5e-06 + | > step_time: 0.1793 (0.2442170945914011) + | > loader_time: 0.0106 (0.011918044169965203) + + + --> TIME: 2024-10-20 00:43:21 -- STEP: 18000/69059 -- GLOBAL_STEP: 18000 + | > loss_text_ce: 0.034543029963970184 (0.05070101377926751) + | > loss_mel_ce: 3.2210378646850586 (3.6277306751012706) + | > loss: 0.8138952255249023 (0.9196079224811652) + | > current_lr: 5e-06 + | > step_time: 0.152 (0.24414315750863816) + | > loader_time: 0.0102 (0.011915289097362069) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_18000.pth + + --> TIME: 2024-10-20 00:43:52 -- STEP: 18050/69059 -- GLOBAL_STEP: 18050 + | > loss_text_ce: 0.039181988686323166 (0.05066698010917996) + | > loss_mel_ce: 3.3011937141418457 (3.6269708296598706) + | > loss: 0.8350939154624939 (0.919409452706492) + | > current_lr: 5e-06 + | > step_time: 0.3541 (0.2441253122002134) + | > loader_time: 0.0113 (0.01191469059096146) + + + --> TIME: 2024-10-20 00:44:11 -- STEP: 18100/69059 -- GLOBAL_STEP: 18100 + | > loss_text_ce: 0.035920269787311554 (0.05063106731136023) + | > loss_mel_ce: 3.1905484199523926 (3.626336677825242) + | > loss: 0.8066172003746033 (0.9192419365492965) + | > current_lr: 5e-06 + | > step_time: 0.1527 (0.24413617995562475) + | > loader_time: 0.0108 (0.011914515021097567) + + + --> TIME: 2024-10-20 00:44:30 -- STEP: 18150/69059 -- GLOBAL_STEP: 18150 + | > loss_text_ce: 0.03611443191766739 (0.05059740379719518) + | > loss_mel_ce: 3.0648515224456787 (3.6256847179953935) + | > loss: 0.775241494178772 (0.919070530719336) + | > current_lr: 5e-06 + | > step_time: 0.1729 (0.2441516057453208) + | > loader_time: 0.0105 (0.01191435302913022) + + + --> TIME: 2024-10-20 00:44:48 -- STEP: 18200/69059 -- GLOBAL_STEP: 18200 + | > loss_text_ce: 0.038649097084999084 (0.05056299272658557) + | > loss_mel_ce: 3.3311054706573486 (3.625042586785085) + | > loss: 0.8424386382102966 (0.9189013951460075) + | > current_lr: 5e-06 + | > step_time: 0.164 (0.24414096817865477) + | > loader_time: 0.0104 (0.011913689361823758) + + + --> TIME: 2024-10-20 00:45:07 -- STEP: 18250/69059 -- GLOBAL_STEP: 18250 + | > loss_text_ce: 0.03809887915849686 (0.05052898833461804) + | > loss_mel_ce: 3.4521803855895996 (3.624414447954245) + | > loss: 0.8725697994232178 (0.9187358593385505) + | > current_lr: 5e-06 + | > step_time: 0.2471 (0.24414094611389997) + | > loader_time: 0.0102 (0.011913416366054563) + + + --> TIME: 2024-10-20 00:45:25 -- STEP: 18300/69059 -- GLOBAL_STEP: 18300 + | > loss_text_ce: 0.04131300374865532 (0.05049438127973022) + | > loss_mel_ce: 3.3089892864227295 (3.623629710101028) + | > loss: 0.8375755548477173 (0.9185310231108476) + | > current_lr: 5e-06 + | > step_time: 0.3154 (0.244108165378779) + | > loader_time: 0.0121 (0.011912187474672884) + + + --> TIME: 2024-10-20 00:45:42 -- STEP: 18350/69059 -- GLOBAL_STEP: 18350 + | > loss_text_ce: 0.037951014935970306 (0.050458064833072136) + | > loss_mel_ce: 3.3617024421691895 (3.6228459900406356) + | > loss: 0.8499133586883545 (0.9183260139814832) + | > current_lr: 5e-06 + | > step_time: 0.2955 (0.2440467941793499) + | > loader_time: 0.0122 (0.011910523627694338) + + + --> TIME: 2024-10-20 00:46:01 -- STEP: 18400/69059 -- GLOBAL_STEP: 18400 + | > loss_text_ce: 0.03860653191804886 (0.0504262174730714) + | > loss_mel_ce: 3.893249750137329 (3.6221839208447304) + | > loss: 0.9829640984535217 (0.9181525348288849) + | > current_lr: 5e-06 + | > step_time: 0.344 (0.24407829033291858) + | > loader_time: 0.0129 (0.011910258363122497) + + + --> TIME: 2024-10-20 00:46:19 -- STEP: 18450/69059 -- GLOBAL_STEP: 18450 + | > loss_text_ce: 0.03845099359750748 (0.05039236606843248) + | > loss_mel_ce: 3.543583631515503 (3.621497147593707) + | > loss: 0.8955086469650269 (0.9179723786629305) + | > current_lr: 5e-06 + | > step_time: 0.388 (0.24408441409185974) + | > loader_time: 0.0134 (0.011908926382297395) + + + --> TIME: 2024-10-20 00:46:40 -- STEP: 18500/69059 -- GLOBAL_STEP: 18500 + | > loss_text_ce: 0.039024900645017624 (0.05035794586043889) + | > loss_mel_ce: 3.4365575313568115 (3.6209154567718396) + | > loss: 0.8688955903053284 (0.9178183509111396) + | > current_lr: 5e-06 + | > step_time: 0.1631 (0.24416059381897384) + | > loader_time: 0.0109 (0.011908709629161914) + + + --> TIME: 2024-10-20 00:46:58 -- STEP: 18550/69059 -- GLOBAL_STEP: 18550 + | > loss_text_ce: 0.03982330858707428 (0.05032524815366417) + | > loss_mel_ce: 3.119875431060791 (3.6202641990178246) + | > loss: 0.789924681186676 (0.9176473620445572) + | > current_lr: 5e-06 + | > step_time: 0.1952 (0.24415094184104322) + | > loader_time: 0.0108 (0.011907529676699549) + + + --> TIME: 2024-10-20 00:47:16 -- STEP: 18600/69059 -- GLOBAL_STEP: 18600 + | > loss_text_ce: 0.038957953453063965 (0.05029183486376379) + | > loss_mel_ce: 3.6138317584991455 (3.619539620286664) + | > loss: 0.91319739818573 (0.9174578640473778) + | > current_lr: 5e-06 + | > step_time: 0.2763 (0.2441588193754996) + | > loader_time: 0.0123 (0.011906107920472312) + + + --> TIME: 2024-10-20 00:47:35 -- STEP: 18650/69059 -- GLOBAL_STEP: 18650 + | > loss_text_ce: 0.041583672165870667 (0.050259690082184866) + | > loss_mel_ce: 3.462670087814331 (3.6189011051315934) + | > loss: 0.8760634660720825 (0.9172901990682119) + | > current_lr: 5e-06 + | > step_time: 0.3791 (0.2441679886135275) + | > loader_time: 0.0126 (0.01190429788492318) + + + --> TIME: 2024-10-20 00:47:53 -- STEP: 18700/69059 -- GLOBAL_STEP: 18700 + | > loss_text_ce: 0.03714291751384735 (0.050226134857073) + | > loss_mel_ce: 3.5910141468048096 (3.618297617524692) + | > loss: 0.9070392847061157 (0.9171309383572096) + | > current_lr: 5e-06 + | > step_time: 0.2177 (0.244126729888712) + | > loader_time: 0.0108 (0.01190309543660615) + + + --> TIME: 2024-10-20 00:48:10 -- STEP: 18750/69059 -- GLOBAL_STEP: 18750 + | > loss_text_ce: 0.034911904484033585 (0.050191530400117076) + | > loss_mel_ce: 3.7153820991516113 (3.61773807580311) + | > loss: 0.9375734925270081 (0.9169824018192283) + | > current_lr: 5e-06 + | > step_time: 0.2779 (0.24407804599761962) + | > loader_time: 0.0122 (0.011901397260030084) + + + --> TIME: 2024-10-20 00:48:29 -- STEP: 18800/69059 -- GLOBAL_STEP: 18800 + | > loss_text_ce: 0.037401121109724045 (0.050159249225670825) + | > loss_mel_ce: 3.381166934967041 (3.6170118289932183) + | > loss: 0.8546420335769653 (0.9167927698156927) + | > current_lr: 5e-06 + | > step_time: 0.1861 (0.24408397464042014) + | > loader_time: 0.0108 (0.011899766072313814) + + + --> TIME: 2024-10-20 00:48:46 -- STEP: 18850/69059 -- GLOBAL_STEP: 18850 + | > loss_text_ce: 0.03603716939687729 (0.05012668915705575) + | > loss_mel_ce: 3.1861073970794678 (3.61627611850869) + | > loss: 0.805536150932312 (0.9166007021725644) + | > current_lr: 5e-06 + | > step_time: 0.2955 (0.24405268352923407) + | > loader_time: 0.012 (0.01189761890340545) + + + --> TIME: 2024-10-20 00:49:04 -- STEP: 18900/69059 -- GLOBAL_STEP: 18900 + | > loss_text_ce: 0.039036884903907776 (0.05009433063348273) + | > loss_mel_ce: 3.1849417686462402 (3.615642575713051) + | > loss: 0.8059946894645691 (0.9164342268402607) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24402519099926823) + | > loader_time: 0.0112 (0.011895975221401778) + + + --> TIME: 2024-10-20 00:49:22 -- STEP: 18950/69059 -- GLOBAL_STEP: 18950 + | > loss_text_ce: 0.0384378582239151 (0.05006221165600406) + | > loss_mel_ce: 3.215752601623535 (3.6149027370337024) + | > loss: 0.8135476112365723 (0.9162412374283826) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.24398226841144008) + | > loader_time: 0.0113 (0.011894106059716037) + + + --> TIME: 2024-10-20 00:49:40 -- STEP: 19000/69059 -- GLOBAL_STEP: 19000 + | > loss_text_ce: 0.03725307434797287 (0.05003040092614898) + | > loss_mel_ce: 3.213367223739624 (3.614272476585277) + | > loss: 0.8126550912857056 (0.9160757196357368) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.2439939417713567) + | > loader_time: 0.0106 (0.011892790606147348) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_19000.pth + + --> TIME: 2024-10-20 00:50:16 -- STEP: 19050/69059 -- GLOBAL_STEP: 19050 + | > loss_text_ce: 0.03966587409377098 (0.049998011742363874) + | > loss_mel_ce: 3.4688193798065186 (3.61363547775688) + | > loss: 0.8771213293075562 (0.915908372624339) + | > current_lr: 5e-06 + | > step_time: 0.3628 (0.24400451327246317) + | > loader_time: 0.0146 (0.011892689932675485) + + + --> TIME: 2024-10-20 00:50:34 -- STEP: 19100/69059 -- GLOBAL_STEP: 19100 + | > loss_text_ce: 0.03823959082365036 (0.049965841959557104) + | > loss_mel_ce: 3.5744035243988037 (3.6131420108785073) + | > loss: 0.9031607508659363 (0.915776963455514) + | > current_lr: 5e-06 + | > step_time: 0.3611 (0.24401278895233314) + | > loader_time: 0.0133 (0.011892677988681481) + + + --> TIME: 2024-10-20 00:50:54 -- STEP: 19150/69059 -- GLOBAL_STEP: 19150 + | > loss_text_ce: 0.035648148506879807 (0.04993400432791922) + | > loss_mel_ce: 3.4656660556793213 (3.6125418715439532) + | > loss: 0.875328540802002 (0.915618969220716) + | > current_lr: 5e-06 + | > step_time: 0.1939 (0.24407155049376325) + | > loader_time: 0.0109 (0.011892919814306496) + + + --> TIME: 2024-10-20 00:51:14 -- STEP: 19200/69059 -- GLOBAL_STEP: 19200 + | > loss_text_ce: 0.03663834184408188 (0.04990347261385352) + | > loss_mel_ce: 3.1180920600891113 (3.6119173869117986) + | > loss: 0.7886825799942017 (0.9154552151262753) + | > current_lr: 5e-06 + | > step_time: 0.1526 (0.2441025783618291) + | > loader_time: 0.0111 (0.011892838540176524) + + + --> TIME: 2024-10-20 00:51:31 -- STEP: 19250/69059 -- GLOBAL_STEP: 19250 + | > loss_text_ce: 0.03643815591931343 (0.04987234257761514) + | > loss_mel_ce: 3.5354671478271484 (3.6112992960075174) + | > loss: 0.8929763436317444 (0.9152929098977663) + | > current_lr: 5e-06 + | > step_time: 0.2834 (0.24405842683222387) + | > loader_time: 0.0121 (0.011892184170809627) + + + --> TIME: 2024-10-20 00:51:51 -- STEP: 19300/69059 -- GLOBAL_STEP: 19300 + | > loss_text_ce: 0.03571275994181633 (0.04984188214195341) + | > loss_mel_ce: 3.5314078330993652 (3.61079005132684) + | > loss: 0.8917801380157471 (0.9151579836119017) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.24411040548215876) + | > loader_time: 0.0106 (0.011891824680288801) + + + --> TIME: 2024-10-20 00:52:09 -- STEP: 19350/69059 -- GLOBAL_STEP: 19350 + | > loss_text_ce: 0.03676120936870575 (0.04980991913790215) + | > loss_mel_ce: 3.807551383972168 (3.610146757736984) + | > loss: 0.9610781669616699 (0.914989169463322) + | > current_lr: 5e-06 + | > step_time: 0.4273 (0.24413068027151338) + | > loader_time: 0.0142 (0.011891036994697474) + + + --> TIME: 2024-10-20 00:52:28 -- STEP: 19400/69059 -- GLOBAL_STEP: 19400 + | > loss_text_ce: 0.03665250539779663 (0.0497800337538582) + | > loss_mel_ce: 3.3449246883392334 (3.6095312364691208) + | > loss: 0.8453943133354187 (0.9148278177952022) + | > current_lr: 5e-06 + | > step_time: 0.1647 (0.24414875050180965) + | > loader_time: 0.0111 (0.011891073885652173) + + + --> TIME: 2024-10-20 00:52:47 -- STEP: 19450/69059 -- GLOBAL_STEP: 19450 + | > loss_text_ce: 0.03619226813316345 (0.04974896809550649) + | > loss_mel_ce: 3.194462299346924 (3.6089379145859954) + | > loss: 0.80766361951828 (0.9146717209068232) + | > current_lr: 5e-06 + | > step_time: 0.2058 (0.24415905878905467) + | > loader_time: 0.0117 (0.011890575119646134) + + + --> TIME: 2024-10-20 00:53:06 -- STEP: 19500/69059 -- GLOBAL_STEP: 19500 + | > loss_text_ce: 0.04018867388367653 (0.049717335188426866) + | > loss_mel_ce: 3.278090000152588 (3.6082190422400346) + | > loss: 0.8295696973800659 (0.914484094595297) + | > current_lr: 5e-06 + | > step_time: 0.2444 (0.24417754779717862) + | > loader_time: 0.0118 (0.011889970119182846) + + + --> TIME: 2024-10-20 00:53:25 -- STEP: 19550/69059 -- GLOBAL_STEP: 19550 + | > loss_text_ce: 0.03794708847999573 (0.04968585738326265) + | > loss_mel_ce: 3.5806076526641846 (3.607652135309955) + | > loss: 0.9046387076377869 (0.9143344984121632) + | > current_lr: 5e-06 + | > step_time: 0.187 (0.24420217794530533) + | > loader_time: 0.0122 (0.011889373140261878) + + + --> TIME: 2024-10-20 00:53:43 -- STEP: 19600/69059 -- GLOBAL_STEP: 19600 + | > loss_text_ce: 0.035701170563697815 (0.04965375407541884) + | > loss_mel_ce: 3.110370397567749 (3.606989735450053) + | > loss: 0.7865179181098938 (0.9141608726218031) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.24416388933755914) + | > loader_time: 0.0105 (0.01188807284345429) + + + --> TIME: 2024-10-20 00:54:03 -- STEP: 19650/69059 -- GLOBAL_STEP: 19650 + | > loss_text_ce: 0.04226359724998474 (0.049622564361485114) + | > loss_mel_ce: 3.7713327407836914 (3.6063648192937032) + | > loss: 0.9533990621566772 (0.9139968461420086) + | > current_lr: 5e-06 + | > step_time: 0.5325 (0.24420660910715583) + | > loader_time: 0.0146 (0.011887814834827653) + + + --> TIME: 2024-10-20 00:54:22 -- STEP: 19700/69059 -- GLOBAL_STEP: 19700 + | > loss_text_ce: 0.04147022217512131 (0.049592824328104454) + | > loss_mel_ce: 3.49430775642395 (3.60584539624034) + | > loss: 0.8839445114135742 (0.9138595553733363) + | > current_lr: 5e-06 + | > step_time: 0.4279 (0.2442486606031505) + | > loader_time: 0.0138 (0.011886709753026784) + + + --> TIME: 2024-10-20 00:54:41 -- STEP: 19750/69059 -- GLOBAL_STEP: 19750 + | > loss_text_ce: 0.038520801812410355 (0.0495621008018526) + | > loss_mel_ce: 3.5798933506011963 (3.6052875206742003) + | > loss: 0.9046035408973694 (0.9137124055910707) + | > current_lr: 5e-06 + | > step_time: 0.4274 (0.24428167194656178) + | > loader_time: 0.0131 (0.011886164411713772) + + + --> TIME: 2024-10-20 00:54:58 -- STEP: 19800/69059 -- GLOBAL_STEP: 19800 + | > loss_text_ce: 0.03562586382031441 (0.04952981218168822) + | > loss_mel_ce: 3.3720924854278564 (3.604549531563354) + | > loss: 0.8519296050071716 (0.9135198361554524) + | > current_lr: 5e-06 + | > step_time: 0.1827 (0.24422049200896062) + | > loader_time: 0.0105 (0.011884630882378745) + + + --> TIME: 2024-10-20 00:55:16 -- STEP: 19850/69059 -- GLOBAL_STEP: 19850 + | > loss_text_ce: 0.03481391817331314 (0.04949991940419821) + | > loss_mel_ce: 3.385420560836792 (3.603900645037432) + | > loss: 0.8550586104393005 (0.9133501413186784) + | > current_lr: 5e-06 + | > step_time: 0.1672 (0.2442187127418422) + | > loader_time: 0.0105 (0.011883641418341693) + + + --> TIME: 2024-10-20 00:55:37 -- STEP: 19900/69059 -- GLOBAL_STEP: 19900 + | > loss_text_ce: 0.037845250219106674 (0.0494717108213983) + | > loss_mel_ce: 3.292024612426758 (3.6032338760845524) + | > loss: 0.8324674367904663 (0.9131763969324335) + | > current_lr: 5e-06 + | > step_time: 0.1644 (0.24428771332879762) + | > loader_time: 0.0104 (0.011883067737272601) + + + --> TIME: 2024-10-20 00:55:54 -- STEP: 19950/69059 -- GLOBAL_STEP: 19950 + | > loss_text_ce: 0.035933684557676315 (0.049441384814567495) + | > loss_mel_ce: 3.3987245559692383 (3.602582017927231) + | > loss: 0.8586645722389221 (0.913005850897695) + | > current_lr: 5e-06 + | > step_time: 0.2033 (0.24425823540317085) + | > loader_time: 0.0129 (0.01188194627451116) + + + --> TIME: 2024-10-20 00:56:13 -- STEP: 20000/69059 -- GLOBAL_STEP: 20000 + | > loss_text_ce: 0.040741901844739914 (0.049411349141318235) + | > loss_mel_ce: 3.51218843460083 (3.601841134238233) + | > loss: 0.8882325887680054 (0.9128131210595362) + | > current_lr: 5e-06 + | > step_time: 0.3974 (0.24425189155340193) + | > loader_time: 0.0137 (0.011880975258350334) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_20000.pth + + --> TIME: 2024-10-20 00:56:47 -- STEP: 20050/69059 -- GLOBAL_STEP: 20050 + | > loss_text_ce: 0.037409912794828415 (0.049382254800984425) + | > loss_mel_ce: 3.550565481185913 (3.601284257384597) + | > loss: 0.89699387550354 (0.9126666282537268) + | > current_lr: 5e-06 + | > step_time: 0.4474 (0.24430189485859097) + | > loader_time: 0.0136 (0.0118813748014835) + + + --> TIME: 2024-10-20 00:57:07 -- STEP: 20100/69059 -- GLOBAL_STEP: 20100 + | > loss_text_ce: 0.039871666580438614 (0.04935284784378758) + | > loss_mel_ce: 3.427009344100952 (3.600729672529201) + | > loss: 0.8667202591896057 (0.9125206303033062) + | > current_lr: 5e-06 + | > step_time: 0.2271 (0.2443350039074077) + | > loader_time: 0.0124 (0.011882313400951748) + + + --> TIME: 2024-10-20 00:57:26 -- STEP: 20150/69059 -- GLOBAL_STEP: 20150 + | > loss_text_ce: 0.039698854088783264 (0.04932204022051446) + | > loss_mel_ce: 3.30329966545105 (3.6000461257006826) + | > loss: 0.835749626159668 (0.9123420416894673) + | > current_lr: 5e-06 + | > step_time: 0.1848 (0.24436899751942448) + | > loader_time: 0.0116 (0.011883581436005737) + + + --> TIME: 2024-10-20 00:57:44 -- STEP: 20200/69059 -- GLOBAL_STEP: 20200 + | > loss_text_ce: 0.04306074231863022 (0.04929321706488642) + | > loss_mel_ce: 3.5682287216186523 (3.599429325254827) + | > loss: 0.9028223752975464 (0.9121806357904229) + | > current_lr: 5e-06 + | > step_time: 0.3945 (0.24434789927879183) + | > loader_time: 0.0138 (0.011882935868631463) + + + --> TIME: 2024-10-20 00:58:04 -- STEP: 20250/69059 -- GLOBAL_STEP: 20250 + | > loss_text_ce: 0.03739296644926071 (0.049264242457110814) + | > loss_mel_ce: 3.385377883911133 (3.5988622060234063) + | > loss: 0.8556926846504211 (0.9120316123314839) + | > current_lr: 5e-06 + | > step_time: 0.3226 (0.24440368412159108) + | > loader_time: 0.0119 (0.011883616141331018) + + + --> TIME: 2024-10-20 00:58:23 -- STEP: 20300/69059 -- GLOBAL_STEP: 20300 + | > loss_text_ce: 0.03584449365735054 (0.049235179180490314) + | > loss_mel_ce: 3.075834274291992 (3.5982332114398083) + | > loss: 0.7779197096824646 (0.911867097865771) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.24439127791691295) + | > loader_time: 0.0104 (0.011882745397501953) + + + --> TIME: 2024-10-20 00:58:40 -- STEP: 20350/69059 -- GLOBAL_STEP: 20350 + | > loss_text_ce: 0.03783687204122543 (0.049206004687092846) + | > loss_mel_ce: 3.4140708446502686 (3.597570944957114) + | > loss: 0.8629769086837769 (0.9116942376205897) + | > current_lr: 5e-06 + | > step_time: 0.2058 (0.24434966853561213) + | > loader_time: 0.0109 (0.011881252354427371) + + + --> TIME: 2024-10-20 00:58:59 -- STEP: 20400/69059 -- GLOBAL_STEP: 20400 + | > loss_text_ce: 0.0340694859623909 (0.04917818669695398) + | > loss_mel_ce: 3.053598165512085 (3.5970713534191443) + | > loss: 0.771916925907135 (0.9115623852406055) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.24438170558097316) + | > loader_time: 0.0103 (0.011881147856805808) + + + --> TIME: 2024-10-20 00:59:19 -- STEP: 20450/69059 -- GLOBAL_STEP: 20450 + | > loss_text_ce: 0.03719036653637886 (0.04915044286586608) + | > loss_mel_ce: 3.459857702255249 (3.5964833282433295) + | > loss: 0.8742620348930359 (0.9114084429874972) + | > current_lr: 5e-06 + | > step_time: 0.4329 (0.24443110175704022) + | > loader_time: 0.014 (0.011881060017350226) + + + --> TIME: 2024-10-20 00:59:37 -- STEP: 20500/69059 -- GLOBAL_STEP: 20500 + | > loss_text_ce: 0.035219766199588776 (0.04912058277491741) + | > loss_mel_ce: 3.2456936836242676 (3.5958560872659353) + | > loss: 0.8202283382415771 (0.9112441677087684) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.24440393759564655) + | > loader_time: 0.0109 (0.011880328224926434) + + + --> TIME: 2024-10-20 00:59:55 -- STEP: 20550/69059 -- GLOBAL_STEP: 20550 + | > loss_text_ce: 0.0372549444437027 (0.04909164749602537) + | > loss_mel_ce: 3.352818250656128 (3.595261531862309) + | > loss: 0.8475183248519897 (0.9110882950380184) + | > current_lr: 5e-06 + | > step_time: 0.2067 (0.2444059864331916) + | > loader_time: 0.0105 (0.011879482826177191) + + + --> TIME: 2024-10-20 01:00:13 -- STEP: 20600/69059 -- GLOBAL_STEP: 20600 + | > loss_text_ce: 0.03918196260929108 (0.04906312960073414) + | > loss_mel_ce: 3.24721360206604 (3.5947192509197397) + | > loss: 0.8215988874435425 (0.9109455953234601) + | > current_lr: 5e-06 + | > step_time: 0.1808 (0.24438637574899544) + | > loader_time: 0.012 (0.01187839439771705) + + + --> TIME: 2024-10-20 01:00:31 -- STEP: 20650/69059 -- GLOBAL_STEP: 20650 + | > loss_text_ce: 0.03723454475402832 (0.049035331098944514) + | > loss_mel_ce: 3.310316562652588 (3.594204849989006) + | > loss: 0.836887776851654 (0.9108100454599452) + | > current_lr: 5e-06 + | > step_time: 0.2347 (0.2443626577571287) + | > loader_time: 0.0116 (0.011877871887447157) + + + --> TIME: 2024-10-20 01:00:49 -- STEP: 20700/69059 -- GLOBAL_STEP: 20700 + | > loss_text_ce: 0.037297919392585754 (0.04900489554268071) + | > loss_mel_ce: 3.2600228786468506 (3.593670770511525) + | > loss: 0.82433021068573 (0.9106689167080287) + | > current_lr: 5e-06 + | > step_time: 0.1523 (0.2443569210416453) + | > loader_time: 0.0105 (0.011876337125105534) + + + --> TIME: 2024-10-20 01:01:07 -- STEP: 20750/69059 -- GLOBAL_STEP: 20750 + | > loss_text_ce: 0.037134554237127304 (0.04897708691468072) + | > loss_mel_ce: 3.319199562072754 (3.5930249825672833) + | > loss: 0.8390835523605347 (0.9105005175693919) + | > current_lr: 5e-06 + | > step_time: 0.2723 (0.2443399955795472) + | > loader_time: 0.0108 (0.011875438977436826) + + + --> TIME: 2024-10-20 01:01:28 -- STEP: 20800/69059 -- GLOBAL_STEP: 20800 + | > loss_text_ce: 0.03897541016340256 (0.04894920294379019) + | > loss_mel_ce: 3.4417014122009277 (3.592407765869901) + | > loss: 0.8701692223548889 (0.9103392424061887) + | > current_lr: 5e-06 + | > step_time: 0.3755 (0.24440049342237985) + | > loader_time: 0.0133 (0.011874929494582668) + + + --> TIME: 2024-10-20 01:01:46 -- STEP: 20850/69059 -- GLOBAL_STEP: 20850 + | > loss_text_ce: 0.03936593979597092 (0.048920353696518426) + | > loss_mel_ce: 3.0506415367126465 (3.5917033570904713) + | > loss: 0.7725018858909607 (0.9101559279005011) + | > current_lr: 5e-06 + | > step_time: 0.3004 (0.24438800313204026) + | > loader_time: 0.0112 (0.011873763356563266) + + + --> TIME: 2024-10-20 01:02:03 -- STEP: 20900/69059 -- GLOBAL_STEP: 20900 + | > loss_text_ce: 0.036073312163352966 (0.04889265323129059) + | > loss_mel_ce: 3.4279606342315674 (3.5910503684504653) + | > loss: 0.866008460521698 (0.9099857556192491) + | > current_lr: 5e-06 + | > step_time: 0.1646 (0.24433809302069925) + | > loader_time: 0.0107 (0.01187225111363605) + + + --> TIME: 2024-10-20 01:02:20 -- STEP: 20950/69059 -- GLOBAL_STEP: 20950 + | > loss_text_ce: 0.03528705611824989 (0.04886420966857292) + | > loss_mel_ce: 2.818721294403076 (3.5905666341075935) + | > loss: 0.7135021090507507 (0.9098577111404658) + | > current_lr: 5e-06 + | > step_time: 0.1852 (0.2443036730591039) + | > loader_time: 0.0108 (0.011870237443782818) + + + --> TIME: 2024-10-20 01:02:39 -- STEP: 21000/69059 -- GLOBAL_STEP: 21000 + | > loss_text_ce: 0.036937739700078964 (0.04883684816202581) + | > loss_mel_ce: 3.2731642723083496 (3.5899786544754293) + | > loss: 0.8275254964828491 (0.9097038758567394) + | > current_lr: 5e-06 + | > step_time: 0.2795 (0.24430024811199733) + | > loader_time: 0.0115 (0.011869183880942193) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_21000.pth + + --> TIME: 2024-10-20 01:03:12 -- STEP: 21050/69059 -- GLOBAL_STEP: 21050 + | > loss_text_ce: 0.03781900182366371 (0.04880771487429196) + | > loss_mel_ce: 3.0777344703674316 (3.589351381032304) + | > loss: 0.7788883447647095 (0.9095397741805905) + | > current_lr: 5e-06 + | > step_time: 0.1707 (0.24428778443370466) + | > loader_time: 0.0104 (0.011868561509102928) + + + --> TIME: 2024-10-20 01:03:30 -- STEP: 21100/69059 -- GLOBAL_STEP: 21100 + | > loss_text_ce: 0.03771611675620079 (0.048780045519490126) + | > loss_mel_ce: 3.3275604248046875 (3.5888302389818247) + | > loss: 0.8413191437721252 (0.9094025713346574) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.24428566885220496) + | > loader_time: 0.0117 (0.01186802763509523) + + + --> TIME: 2024-10-20 01:03:50 -- STEP: 21150/69059 -- GLOBAL_STEP: 21150 + | > loss_text_ce: 0.031711895018815994 (0.04875157685717007) + | > loss_mel_ce: 3.442939281463623 (3.5882271147225104) + | > loss: 0.8686627745628357 (0.9092446731027412) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.2443244505943136) + | > loader_time: 0.0111 (0.011867850057904018) + + + --> TIME: 2024-10-20 01:04:10 -- STEP: 21200/69059 -- GLOBAL_STEP: 21200 + | > loss_text_ce: 0.03451785817742348 (0.04872455122081095) + | > loss_mel_ce: 3.1333484649658203 (3.587695392010337) + | > loss: 0.7919665575027466 (0.9091049860136682) + | > current_lr: 5e-06 + | > step_time: 0.1725 (0.24438723083936942) + | > loader_time: 0.0109 (0.011868510212538354) + + + --> TIME: 2024-10-20 01:04:30 -- STEP: 21250/69059 -- GLOBAL_STEP: 21250 + | > loss_text_ce: 0.03733832389116287 (0.04869664803278786) + | > loss_mel_ce: 3.3172245025634766 (3.587051904375403) + | > loss: 0.8386406898498535 (0.9089371383021853) + | > current_lr: 5e-06 + | > step_time: 0.2779 (0.24441362993576948) + | > loader_time: 0.0121 (0.011869353417789214) + + + --> TIME: 2024-10-20 01:04:47 -- STEP: 21300/69059 -- GLOBAL_STEP: 21300 + | > loss_text_ce: 0.03530687093734741 (0.048669881374430804) + | > loss_mel_ce: 3.273956775665283 (3.5863741698287366) + | > loss: 0.8273159265518188 (0.9087610130019025) + | > current_lr: 5e-06 + | > step_time: 0.181 (0.2443725221033947) + | > loader_time: 0.0106 (0.0118679322547196) + + + --> TIME: 2024-10-20 01:05:06 -- STEP: 21350/69059 -- GLOBAL_STEP: 21350 + | > loss_text_ce: 0.036131929606199265 (0.04864424953302598) + | > loss_mel_ce: 3.058485269546509 (3.5857937021612747) + | > loss: 0.7736542820930481 (0.9086094881304526) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24439921924046107) + | > loader_time: 0.0106 (0.01186724510907566) + + + --> TIME: 2024-10-20 01:05:24 -- STEP: 21400/69059 -- GLOBAL_STEP: 21400 + | > loss_text_ce: 0.03675108775496483 (0.04861770100900283) + | > loss_mel_ce: 3.026124954223633 (3.585165618635774) + | > loss: 0.7657189965248108 (0.9084458301084057) + | > current_lr: 5e-06 + | > step_time: 0.1994 (0.2443781102372107) + | > loader_time: 0.0122 (0.0118664210310606) + + + --> TIME: 2024-10-20 01:05:43 -- STEP: 21450/69059 -- GLOBAL_STEP: 21450 + | > loss_text_ce: 0.0362919382750988 (0.04859084266700309) + | > loss_mel_ce: 3.5602102279663086 (3.58457199457919) + | > loss: 0.8991255164146423 (0.9082907095183297) + | > current_lr: 5e-06 + | > step_time: 0.2355 (0.2443941294952428) + | > loader_time: 0.0116 (0.011865873725819953) + + + --> TIME: 2024-10-20 01:06:01 -- STEP: 21500/69059 -- GLOBAL_STEP: 21500 + | > loss_text_ce: 0.03871599957346916 (0.048565083947725664) + | > loss_mel_ce: 3.2603960037231445 (3.5840049426721876) + | > loss: 0.8247780203819275 (0.9081425068766565) + | > current_lr: 5e-06 + | > step_time: 0.2596 (0.24440244487274526) + | > loader_time: 0.0114 (0.01186495975006456) + + + --> TIME: 2024-10-20 01:06:19 -- STEP: 21550/69059 -- GLOBAL_STEP: 21550 + | > loss_text_ce: 0.03626853972673416 (0.04853962966977779) + | > loss_mel_ce: 3.2315211296081543 (3.5835265184167824) + | > loss: 0.8169474005699158 (0.9080165372315243) + | > current_lr: 5e-06 + | > step_time: 0.1959 (0.24437949688418006) + | > loader_time: 0.0107 (0.01186342241592581) + + + --> TIME: 2024-10-20 01:06:36 -- STEP: 21600/69059 -- GLOBAL_STEP: 21600 + | > loss_text_ce: 0.037771351635456085 (0.04851372964481847) + | > loss_mel_ce: 3.3356051445007324 (3.582897910828934) + | > loss: 0.8433441519737244 (0.9078529103293457) + | > current_lr: 5e-06 + | > step_time: 0.1575 (0.24432948696392554) + | > loader_time: 0.0103 (0.011862861370598802) + + + --> TIME: 2024-10-20 01:06:55 -- STEP: 21650/69059 -- GLOBAL_STEP: 21650 + | > loss_text_ce: 0.03725669905543327 (0.04848683342707546) + | > loss_mel_ce: 3.3572630882263184 (3.582396347076724) + | > loss: 0.8486299514770508 (0.907720795338478) + | > current_lr: 5e-06 + | > step_time: 0.2713 (0.24435154166012382) + | > loader_time: 0.0102 (0.011862085950291984) + + + --> TIME: 2024-10-20 01:07:13 -- STEP: 21700/69059 -- GLOBAL_STEP: 21700 + | > loss_text_ce: 0.03556784614920616 (0.048459935615671704) + | > loss_mel_ce: 3.301360845565796 (3.581807352035267) + | > loss: 0.8342321515083313 (0.9075668221190222) + | > current_lr: 5e-06 + | > step_time: 0.1807 (0.24430749292197865) + | > loader_time: 0.011 (0.011860648091487595) + + + --> TIME: 2024-10-20 01:07:32 -- STEP: 21750/69059 -- GLOBAL_STEP: 21750 + | > loss_text_ce: 0.038183849304914474 (0.04843372043461967) + | > loss_mel_ce: 3.413907527923584 (3.5813333622833685) + | > loss: 0.8630228638648987 (0.9074417708824414) + | > current_lr: 5e-06 + | > step_time: 0.2311 (0.24433550199969062) + | > loader_time: 0.0109 (0.011860455129338381) + + + --> TIME: 2024-10-20 01:07:50 -- STEP: 21800/69059 -- GLOBAL_STEP: 21800 + | > loss_text_ce: 0.038560494780540466 (0.048408375009467056) + | > loss_mel_ce: 3.1871495246887207 (3.580761484574825) + | > loss: 0.8064274787902832 (0.9072924650945788) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.2443263344152258) + | > loader_time: 0.0107 (0.011858612156789211) + + + --> TIME: 2024-10-20 01:08:09 -- STEP: 21850/69059 -- GLOBAL_STEP: 21850 + | > loss_text_ce: 0.035768456757068634 (0.04838357403286183) + | > loss_mel_ce: 3.212991237640381 (3.580251133785631) + | > loss: 0.8121899366378784 (0.9071586771551471) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.2443387884406258) + | > loader_time: 0.0108 (0.011857726557576657) + + + --> TIME: 2024-10-20 01:08:27 -- STEP: 21900/69059 -- GLOBAL_STEP: 21900 + | > loss_text_ce: 0.040700092911720276 (0.048358665673002565) + | > loss_mel_ce: 3.524583101272583 (3.579587228646552) + | > loss: 0.8913208246231079 (0.9069864737796994) + | > current_lr: 5e-06 + | > step_time: 0.1573 (0.24433395371589486) + | > loader_time: 0.0105 (0.011856238460976214) + + + --> TIME: 2024-10-20 01:08:44 -- STEP: 21950/69059 -- GLOBAL_STEP: 21950 + | > loss_text_ce: 0.037707384675741196 (0.0483330858993536) + | > loss_mel_ce: 2.871713876724243 (3.5788874116628113) + | > loss: 0.7273553013801575 (0.9068051246005466) + | > current_lr: 5e-06 + | > step_time: 0.2265 (0.24428844562695617) + | > loader_time: 0.0114 (0.011855351843432007) + + + --> TIME: 2024-10-20 01:09:02 -- STEP: 22000/69059 -- GLOBAL_STEP: 22000 + | > loss_text_ce: 0.03687618300318718 (0.048307082178325036) + | > loss_mel_ce: 3.5419676303863525 (3.578336512587278) + | > loss: 0.8947109580039978 (0.9066608988994895) + | > current_lr: 5e-06 + | > step_time: 0.2362 (0.2442402556592768) + | > loader_time: 0.0122 (0.011854006583040385) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_22000.pth + + --> TIME: 2024-10-20 01:09:34 -- STEP: 22050/69059 -- GLOBAL_STEP: 22050 + | > loss_text_ce: 0.041251812130212784 (0.04828124027564064) + | > loss_mel_ce: 3.4871714115142822 (3.5777366094848646) + | > loss: 0.882105827331543 (0.9065044626492215) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.2442064952201584) + | > loader_time: 0.0107 (0.011853174795639462) + + + --> TIME: 2024-10-20 01:09:53 -- STEP: 22100/69059 -- GLOBAL_STEP: 22100 + | > loss_text_ce: 0.03750232979655266 (0.048255112990315444) + | > loss_mel_ce: 3.7954249382019043 (3.5771804483875336) + | > loss: 0.9582318067550659 (0.9063588905523259) + | > current_lr: 5e-06 + | > step_time: 0.4998 (0.24421922996572779) + | > loader_time: 0.015 (0.011852667881892253) + + + --> TIME: 2024-10-20 01:10:12 -- STEP: 22150/69059 -- GLOBAL_STEP: 22150 + | > loss_text_ce: 0.03531116619706154 (0.048230079316174505) + | > loss_mel_ce: 3.6473307609558105 (3.5767015686960755) + | > loss: 0.9206604957580566 (0.9062329122148296) + | > current_lr: 5e-06 + | > step_time: 0.2051 (0.24424224056993057) + | > loader_time: 0.0115 (0.011853211954000546) + + + --> TIME: 2024-10-20 01:10:31 -- STEP: 22200/69059 -- GLOBAL_STEP: 22200 + | > loss_text_ce: 0.03982109948992729 (0.04820412392224603) + | > loss_mel_ce: 3.6128122806549072 (3.5762100821464893) + | > loss: 0.9131583571434021 (0.9061035517260828) + | > current_lr: 5e-06 + | > step_time: 0.1906 (0.24426509935576637) + | > loader_time: 0.0124 (0.011853183958981463) + + + --> TIME: 2024-10-20 01:10:51 -- STEP: 22250/69059 -- GLOBAL_STEP: 22250 + | > loss_text_ce: 0.037820249795913696 (0.04817985498193599) + | > loss_mel_ce: 3.560276985168457 (3.575646195411673) + | > loss: 0.8995243310928345 (0.905956512802102) + | > current_lr: 5e-06 + | > step_time: 0.1848 (0.24432136823889913) + | > loader_time: 0.0104 (0.011853831141182521) + + + --> TIME: 2024-10-20 01:11:12 -- STEP: 22300/69059 -- GLOBAL_STEP: 22300 + | > loss_text_ce: 0.03752823546528816 (0.04815477092061265) + | > loss_mel_ce: 3.6440651416778564 (3.575227021133944) + | > loss: 0.9203983545303345 (0.9058454482143763) + | > current_lr: 5e-06 + | > step_time: 0.3946 (0.2443944688335128) + | > loader_time: 0.0132 (0.011853471326186492) + + + --> TIME: 2024-10-20 01:11:30 -- STEP: 22350/69059 -- GLOBAL_STEP: 22350 + | > loss_text_ce: 0.03599914535880089 (0.04813056019470252) + | > loss_mel_ce: 3.5669357776641846 (3.57467875036883) + | > loss: 0.9007337093353271 (0.9057023278392128) + | > current_lr: 5e-06 + | > step_time: 0.5056 (0.2443735170044355) + | > loader_time: 0.0141 (0.01185238808326804) + + + --> TIME: 2024-10-20 01:11:49 -- STEP: 22400/69059 -- GLOBAL_STEP: 22400 + | > loss_text_ce: 0.03698137030005455 (0.04810649102046477) + | > loss_mel_ce: 3.3564131259918213 (3.5741072059635512) + | > loss: 0.8483486175537109 (0.905553424435534) + | > current_lr: 5e-06 + | > step_time: 0.1968 (0.24437494949570723) + | > loader_time: 0.0109 (0.011852668055466219) + + + --> TIME: 2024-10-20 01:12:07 -- STEP: 22450/69059 -- GLOBAL_STEP: 22450 + | > loss_text_ce: 0.03437868505716324 (0.04808153795737987) + | > loss_mel_ce: 3.3945915699005127 (3.5735624955969465) + | > loss: 0.8572425842285156 (0.9054110085778346) + | > current_lr: 5e-06 + | > step_time: 0.1951 (0.24435825386132323) + | > loader_time: 0.011 (0.011852163132155633) + + + --> TIME: 2024-10-20 01:12:25 -- STEP: 22500/69059 -- GLOBAL_STEP: 22500 + | > loss_text_ce: 0.03708991780877113 (0.04805687559147666) + | > loss_mel_ce: 2.939255714416504 (3.573084271399171) + | > loss: 0.7440863847732544 (0.9052852869272225) + | > current_lr: 5e-06 + | > step_time: 0.1638 (0.24434764136208428) + | > loader_time: 0.0103 (0.01185155303743148) + + + --> TIME: 2024-10-20 01:12:45 -- STEP: 22550/69059 -- GLOBAL_STEP: 22550 + | > loss_text_ce: 0.03495000675320625 (0.04803240958708607) + | > loss_mel_ce: 3.206627368927002 (3.572599333490333) + | > loss: 0.8103943467140198 (0.9051579359550427) + | > current_lr: 5e-06 + | > step_time: 0.1872 (0.2444236840377097) + | > loader_time: 0.011 (0.011852010750189036) + + + --> TIME: 2024-10-20 01:13:04 -- STEP: 22600/69059 -- GLOBAL_STEP: 22600 + | > loss_text_ce: 0.03566034883260727 (0.048008603334789415) + | > loss_mel_ce: 3.3678247928619385 (3.5721012491884396) + | > loss: 0.8508712649345398 (0.9050274633174442) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.24442958560665096) + | > loader_time: 0.0109 (0.011851226555562627) + + + --> TIME: 2024-10-20 01:13:24 -- STEP: 22650/69059 -- GLOBAL_STEP: 22650 + | > loss_text_ce: 0.03667161986231804 (0.04798350022525967) + | > loss_mel_ce: 3.1017167568206787 (3.571604486143343) + | > loss: 0.7845970988273621 (0.9048969967817886) + | > current_lr: 5e-06 + | > step_time: 0.2795 (0.24446664729128872) + | > loader_time: 0.011 (0.011851360897881018) + + + --> TIME: 2024-10-20 01:13:41 -- STEP: 22700/69059 -- GLOBAL_STEP: 22700 + | > loss_text_ce: 0.03449272736907005 (0.04795847988059726) + | > loss_mel_ce: 3.4815175533294678 (3.5710842912417586) + | > loss: 0.879002571105957 (0.9047606929598394) + | > current_lr: 5e-06 + | > step_time: 0.1577 (0.2444332199264728) + | > loader_time: 0.0103 (0.011850872680479184) + + + --> TIME: 2024-10-20 01:13:58 -- STEP: 22750/69059 -- GLOBAL_STEP: 22750 + | > loss_text_ce: 0.037975918501615524 (0.04793386040136699) + | > loss_mel_ce: 3.2404651641845703 (3.5705205038301147) + | > loss: 0.8196102976799011 (0.9046135912439319) + | > current_lr: 5e-06 + | > step_time: 0.2058 (0.2443842636988713) + | > loader_time: 0.0107 (0.011849379497569971) + + + --> TIME: 2024-10-20 01:14:16 -- STEP: 22800/69059 -- GLOBAL_STEP: 22800 + | > loss_text_ce: 0.036722421646118164 (0.04790941533844974) + | > loss_mel_ce: 3.360280752182007 (3.569925716707572) + | > loss: 0.8492507934570312 (0.9044587832009576) + | > current_lr: 5e-06 + | > step_time: 0.23 (0.24434962763075244) + | > loader_time: 0.0115 (0.011848165351047825) + + + --> TIME: 2024-10-20 01:14:35 -- STEP: 22850/69059 -- GLOBAL_STEP: 22850 + | > loss_text_ce: 0.03714568167924881 (0.0478856895839554) + | > loss_mel_ce: 3.1685941219329834 (3.5694988832118906) + | > loss: 0.8014349341392517 (0.90434614339073) + | > current_lr: 5e-06 + | > step_time: 0.2157 (0.24436987969755344) + | > loader_time: 0.0116 (0.011848241390754124) + + + --> TIME: 2024-10-20 01:14:53 -- STEP: 22900/69059 -- GLOBAL_STEP: 22900 + | > loss_text_ce: 0.03342273458838463 (0.04786179187565251) + | > loss_mel_ce: 3.1700057983398438 (3.56889916726074) + | > loss: 0.8008571267127991 (0.9041902399661753) + | > current_lr: 5e-06 + | > step_time: 0.1577 (0.24434912063148864) + | > loader_time: 0.0103 (0.011846910949357187) + + + --> TIME: 2024-10-20 01:15:12 -- STEP: 22950/69059 -- GLOBAL_STEP: 22950 + | > loss_text_ce: 0.036683276295661926 (0.04783657053846669) + | > loss_mel_ce: 3.1844568252563477 (3.5683873877597962) + | > loss: 0.8052850365638733 (0.9040559897495505) + | > current_lr: 5e-06 + | > step_time: 0.215 (0.24436941065819434) + | > loader_time: 0.0111 (0.011846469534248543) + + + --> TIME: 2024-10-20 01:15:30 -- STEP: 23000/69059 -- GLOBAL_STEP: 23000 + | > loss_text_ce: 0.035373102873563766 (0.047811705283172655) + | > loss_mel_ce: 3.123767137527466 (3.5678064412655948) + | > loss: 0.7897850871086121 (0.9039045368199756) + | > current_lr: 5e-06 + | > step_time: 0.1633 (0.24436960390339726) + | > loader_time: 0.0103 (0.011845417665398615) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_23000.pth + + --> TIME: 2024-10-20 01:16:03 -- STEP: 23050/69059 -- GLOBAL_STEP: 23050 + | > loss_text_ce: 0.03674852475523949 (0.04778757617814929) + | > loss_mel_ce: 3.286041498184204 (3.5673077813042906) + | > loss: 0.8306974768638611 (0.9037738395435426) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.24436089177452303) + | > loader_time: 0.0106 (0.011845029336511418) + + + --> TIME: 2024-10-20 01:16:23 -- STEP: 23100/69059 -- GLOBAL_STEP: 23100 + | > loss_text_ce: 0.038496654480695724 (0.04776491983467098) + | > loss_mel_ce: 3.447406768798828 (3.5668833832203997) + | > loss: 0.871475875377655 (0.903662075939632) + | > current_lr: 5e-06 + | > step_time: 0.3695 (0.24439723296598956) + | > loader_time: 0.0133 (0.01184474558025209) + + + --> TIME: 2024-10-20 01:16:43 -- STEP: 23150/69059 -- GLOBAL_STEP: 23150 + | > loss_text_ce: 0.04016602784395218 (0.04774143514008157) + | > loss_mel_ce: 3.413515090942383 (3.5663456780452103) + | > loss: 0.863420307636261 (0.9035217784728132) + | > current_lr: 5e-06 + | > step_time: 0.4443 (0.24443905906594854) + | > loader_time: 0.0134 (0.011844542629002985) + + + --> TIME: 2024-10-20 01:17:03 -- STEP: 23200/69059 -- GLOBAL_STEP: 23200 + | > loss_text_ce: 0.0345158725976944 (0.04771776389202157) + | > loss_mel_ce: 3.4213359355926514 (3.5659299460978255) + | > loss: 0.8639629483222961 (0.903411927672809) + | > current_lr: 5e-06 + | > step_time: 0.1743 (0.24448747704769003) + | > loader_time: 0.0108 (0.0118450050312897) + + + --> TIME: 2024-10-20 01:17:21 -- STEP: 23250/69059 -- GLOBAL_STEP: 23250 + | > loss_text_ce: 0.03691211715340614 (0.04769432944315726) + | > loss_mel_ce: 3.277252435684204 (3.565414660299969) + | > loss: 0.8285411596298218 (0.9032772476083483) + | > current_lr: 5e-06 + | > step_time: 0.1963 (0.24447766564482) + | > loader_time: 0.0107 (0.011844606368772421) + + + --> TIME: 2024-10-20 01:17:39 -- STEP: 23300/69059 -- GLOBAL_STEP: 23300 + | > loss_text_ce: 0.03955545276403427 (0.0476708010152429) + | > loss_mel_ce: 3.60939884185791 (3.5648936445723187) + | > loss: 0.912238597869873 (0.9031411115780402) + | > current_lr: 5e-06 + | > step_time: 0.5134 (0.24447295382299136) + | > loader_time: 0.0153 (0.011843937875886818) + + + --> TIME: 2024-10-20 01:17:56 -- STEP: 23350/69059 -- GLOBAL_STEP: 23350 + | > loss_text_ce: 0.03613046929240227 (0.04764627439434565) + | > loss_mel_ce: 3.4717142581939697 (3.564307781000966) + | > loss: 0.8769611716270447 (0.902988514032302) + | > current_lr: 5e-06 + | > step_time: 0.2109 (0.2444273975115006) + | > loader_time: 0.0106 (0.011842816371223376) + + + --> TIME: 2024-10-20 01:18:15 -- STEP: 23400/69059 -- GLOBAL_STEP: 23400 + | > loss_text_ce: 0.0350726880133152 (0.04762416516374946) + | > loss_mel_ce: 3.0688021183013916 (3.5638448992651663) + | > loss: 0.7759687304496765 (0.902867266298868) + | > current_lr: 5e-06 + | > step_time: 0.1566 (0.24442888447362135) + | > loader_time: 0.0102 (0.01184224301933221) + + + --> TIME: 2024-10-20 01:18:34 -- STEP: 23450/69059 -- GLOBAL_STEP: 23450 + | > loss_text_ce: 0.03797778859734535 (0.04760094583495202) + | > loss_mel_ce: 3.317639112472534 (3.5633054286533747) + | > loss: 0.8389042019844055 (0.9027265938156951) + | > current_lr: 5e-06 + | > step_time: 0.2266 (0.244450070446234) + | > loader_time: 0.0117 (0.01184233577012505) + + + --> TIME: 2024-10-20 01:18:52 -- STEP: 23500/69059 -- GLOBAL_STEP: 23500 + | > loss_text_ce: 0.0394577831029892 (0.04757699356592707) + | > loss_mel_ce: 3.321207284927368 (3.5626916039142116) + | > loss: 0.8401662707328796 (0.902567149563038) + | > current_lr: 5e-06 + | > step_time: 0.3024 (0.24442524404728666) + | > loader_time: 0.0128 (0.011841248968814258) + + + --> TIME: 2024-10-20 01:19:10 -- STEP: 23550/69059 -- GLOBAL_STEP: 23550 + | > loss_text_ce: 0.03310687467455864 (0.047553427345781725) + | > loss_mel_ce: 3.189724922180176 (3.5621258805613016) + | > loss: 0.8057079315185547 (0.9024198271792906) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.24439966375913844) + | > loader_time: 0.0106 (0.01184016386667885) + + + --> TIME: 2024-10-20 01:19:29 -- STEP: 23600/69059 -- GLOBAL_STEP: 23600 + | > loss_text_ce: 0.03913109749555588 (0.04753108025354844) + | > loss_mel_ce: 3.4894087314605713 (3.5616515624624103) + | > loss: 0.8821349740028381 (0.9022956608841979) + | > current_lr: 5e-06 + | > step_time: 0.1635 (0.24443515645245376) + | > loader_time: 0.0111 (0.011839379567210933) + + + --> TIME: 2024-10-20 01:19:48 -- STEP: 23650/69059 -- GLOBAL_STEP: 23650 + | > loss_text_ce: 0.03654875606298447 (0.0475078009580897) + | > loss_mel_ce: 3.3346173763275146 (3.561141027244898) + | > loss: 0.8427915573120117 (0.9021622072591836) + | > current_lr: 5e-06 + | > step_time: 0.1853 (0.24443786961835737) + | > loader_time: 0.0111 (0.011838923458317094) + + + --> TIME: 2024-10-20 01:20:07 -- STEP: 23700/69059 -- GLOBAL_STEP: 23700 + | > loss_text_ce: 0.03617098182439804 (0.047484389298481436) + | > loss_mel_ce: 3.6033835411071777 (3.5606535959344288) + | > loss: 0.9098886251449585 (0.9020344965191324) + | > current_lr: 5e-06 + | > step_time: 0.3702 (0.24445233012050516) + | > loader_time: 0.0131 (0.011837878589388666) + + + --> TIME: 2024-10-20 01:20:26 -- STEP: 23750/69059 -- GLOBAL_STEP: 23750 + | > loss_text_ce: 0.036101143807172775 (0.04746204186521069) + | > loss_mel_ce: 3.3667521476745605 (3.5601367460953477) + | > loss: 0.850713312625885 (0.9018996972084039) + | > current_lr: 5e-06 + | > step_time: 0.2837 (0.24446862419529966) + | > loader_time: 0.0122 (0.011837033653259254) + + + --> TIME: 2024-10-20 01:20:45 -- STEP: 23800/69059 -- GLOBAL_STEP: 23800 + | > loss_text_ce: 0.038249671459198 (0.04743928640227787) + | > loss_mel_ce: 3.5019445419311523 (3.5595874455095253) + | > loss: 0.8850485682487488 (0.9017566831973415) + | > current_lr: 5e-06 + | > step_time: 0.3576 (0.24450858396642347) + | > loader_time: 0.0132 (0.011836860750903578) + + + --> TIME: 2024-10-20 01:21:04 -- STEP: 23850/69059 -- GLOBAL_STEP: 23850 + | > loss_text_ce: 0.035323675721883774 (0.04741671072073131) + | > loss_mel_ce: 3.33601975440979 (3.5590742942422104) + | > loss: 0.8428358435630798 (0.901622751456386) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.24451077489233117) + | > loader_time: 0.0128 (0.01183590018274397) + + + --> TIME: 2024-10-20 01:21:23 -- STEP: 23900/69059 -- GLOBAL_STEP: 23900 + | > loss_text_ce: 0.03873978555202484 (0.047393557082269264) + | > loss_mel_ce: 3.147528886795044 (3.5586231138516586) + | > loss: 0.7965671420097351 (0.901504167944317) + | > current_lr: 5e-06 + | > step_time: 0.1777 (0.24454157890136272) + | > loader_time: 0.0107 (0.011835234085386245) + + + --> TIME: 2024-10-20 01:21:42 -- STEP: 23950/69059 -- GLOBAL_STEP: 23950 + | > loss_text_ce: 0.03616591542959213 (0.047371422721768995) + | > loss_mel_ce: 3.3437347412109375 (3.558104062996221) + | > loss: 0.8449751734733582 (0.9013688716385702) + | > current_lr: 5e-06 + | > step_time: 0.3951 (0.24453137888540058) + | > loader_time: 0.0136 (0.01183409865066749) + + + --> TIME: 2024-10-20 01:21:59 -- STEP: 24000/69059 -- GLOBAL_STEP: 24000 + | > loss_text_ce: 0.03488560765981674 (0.0473480893051406) + | > loss_mel_ce: 3.3592112064361572 (3.5575438951551828) + | > loss: 0.8485242128372192 (0.9012229963218167) + | > current_lr: 5e-06 + | > step_time: 0.2313 (0.24449496962626774) + | > loader_time: 0.0118 (0.011833106150229748) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_24000.pth + + --> TIME: 2024-10-20 01:22:33 -- STEP: 24050/69059 -- GLOBAL_STEP: 24050 + | > loss_text_ce: 0.03440447524189949 (0.047325534760378776) + | > loss_mel_ce: 3.1591315269470215 (3.5570324845125585) + | > loss: 0.7983840107917786 (0.9010895050265927) + | > current_lr: 5e-06 + | > step_time: 0.2053 (0.24450839913064873) + | > loader_time: 0.0112 (0.011832602970813243) + + + --> TIME: 2024-10-20 01:22:51 -- STEP: 24100/69059 -- GLOBAL_STEP: 24100 + | > loss_text_ce: 0.03506644815206528 (0.04730276673251591) + | > loss_mel_ce: 3.2201712131500244 (3.5564884550838465) + | > loss: 0.8138093948364258 (0.9009478056529739) + | > current_lr: 5e-06 + | > step_time: 0.1648 (0.244492605207372) + | > loader_time: 0.0115 (0.011832784547845334) + + + --> TIME: 2024-10-20 01:23:09 -- STEP: 24150/69059 -- GLOBAL_STEP: 24150 + | > loss_text_ce: 0.035043299198150635 (0.04728038226788388) + | > loss_mel_ce: 3.0802500247955322 (3.556002204472472) + | > loss: 0.7788233160972595 (0.9008206468808225) + | > current_lr: 5e-06 + | > step_time: 0.1787 (0.24447912978336184) + | > loader_time: 0.011 (0.011832433003569712) + + + --> TIME: 2024-10-20 01:23:28 -- STEP: 24200/69059 -- GLOBAL_STEP: 24200 + | > loss_text_ce: 0.03700244054198265 (0.04725805841024555) + | > loss_mel_ce: 3.373396396636963 (3.5554730031115866) + | > loss: 0.8525996804237366 (0.9006827655680902) + | > current_lr: 5e-06 + | > step_time: 0.1641 (0.24448385082000543) + | > loader_time: 0.0102 (0.011831904157134103) + + + --> TIME: 2024-10-20 01:23:47 -- STEP: 24250/69059 -- GLOBAL_STEP: 24250 + | > loss_text_ce: 0.03548332303762436 (0.04723725175066391) + | > loss_mel_ce: 3.093811511993408 (3.5549662015954158) + | > loss: 0.7823237180709839 (0.9005508635216146) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.24450811574876924) + | > loader_time: 0.0105 (0.011831458829112865) + + + --> TIME: 2024-10-20 01:24:04 -- STEP: 24300/69059 -- GLOBAL_STEP: 24300 + | > loss_text_ce: 0.0368756428360939 (0.047214543230969505) + | > loss_mel_ce: 3.2295279502868652 (3.5544131630928835) + | > loss: 0.8166009187698364 (0.9004069267624194) + | > current_lr: 5e-06 + | > step_time: 0.197 (0.24447210626837648) + | > loader_time: 0.0109 (0.01183087214520927) + + + --> TIME: 2024-10-20 01:24:21 -- STEP: 24350/69059 -- GLOBAL_STEP: 24350 + | > loss_text_ce: 0.034489162266254425 (0.047191585635793065) + | > loss_mel_ce: 3.515174627304077 (3.5538057904370675) + | > loss: 0.8874159455299377 (0.9002493441912667) + | > current_lr: 5e-06 + | > step_time: 0.2227 (0.24440792640866196) + | > loader_time: 0.0108 (0.011829323670702526) + + + --> TIME: 2024-10-20 01:24:38 -- STEP: 24400/69059 -- GLOBAL_STEP: 24400 + | > loss_text_ce: 0.03640059381723404 (0.0471682510447886) + | > loss_mel_ce: 3.2132062911987305 (3.553290393538154) + | > loss: 0.8124017119407654 (0.9001146613232419) + | > current_lr: 5e-06 + | > step_time: 0.1559 (0.24437561705464222) + | > loader_time: 0.0106 (0.011827836456845995) + + + --> TIME: 2024-10-20 01:24:57 -- STEP: 24450/69059 -- GLOBAL_STEP: 24450 + | > loss_text_ce: 0.03673515096306801 (0.0471462061215793) + | > loss_mel_ce: 3.1446712017059326 (3.552822412732668) + | > loss: 0.7953515648841858 (0.8999921548951617) + | > current_lr: 5e-06 + | > step_time: 0.3162 (0.24437609900726132) + | > loader_time: 0.0118 (0.01182711619784737) + + + --> TIME: 2024-10-20 01:25:17 -- STEP: 24500/69059 -- GLOBAL_STEP: 24500 + | > loss_text_ce: 0.03660279139876366 (0.04712476900402371) + | > loss_mel_ce: 3.09631085395813 (3.55238828859523) + | > loss: 0.7832283973693848 (0.8998782645707222) + | > current_lr: 5e-06 + | > step_time: 0.3718 (0.24444520450124935) + | > loader_time: 0.0132 (0.011827004666231093) + + + --> TIME: 2024-10-20 01:25:35 -- STEP: 24550/69059 -- GLOBAL_STEP: 24550 + | > loss_text_ce: 0.0364391915500164 (0.047102948799729254) + | > loss_mel_ce: 3.1950695514678955 (3.5518449358541866) + | > loss: 0.8078771829605103 (0.8997369713258835) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24441027757835) + | > loader_time: 0.0101 (0.011825885627031789) + + + --> TIME: 2024-10-20 01:25:53 -- STEP: 24600/69059 -- GLOBAL_STEP: 24600 + | > loss_text_ce: 0.0360102653503418 (0.04708192789336521) + | > loss_mel_ce: 3.083688259124756 (3.551297877862193) + | > loss: 0.7799246311187744 (0.8995949515966859) + | > current_lr: 5e-06 + | > step_time: 0.3161 (0.24440681173549436) + | > loader_time: 0.0125 (0.01182462202824226) + + + --> TIME: 2024-10-20 01:26:11 -- STEP: 24650/69059 -- GLOBAL_STEP: 24650 + | > loss_text_ce: 0.034280646592378616 (0.047059670896486536) + | > loss_mel_ce: 3.2724225521087646 (3.550767839733526) + | > loss: 0.8266757726669312 (0.899456877807574) + | > current_lr: 5e-06 + | > step_time: 0.2113 (0.24438543527653445) + | > loader_time: 0.0108 (0.01182332011797363) + + + --> TIME: 2024-10-20 01:26:29 -- STEP: 24700/69059 -- GLOBAL_STEP: 24700 + | > loss_text_ce: 0.037184327840805054 (0.0470376770392904) + | > loss_mel_ce: 3.353785753250122 (3.5502904128352557) + | > loss: 0.84774249792099 (0.8993320226162546) + | > current_lr: 5e-06 + | > step_time: 0.1972 (0.24435668485849976) + | > loader_time: 0.0109 (0.011822507352481464) + + + --> TIME: 2024-10-20 01:26:45 -- STEP: 24750/69059 -- GLOBAL_STEP: 24750 + | > loss_text_ce: 0.03520941361784935 (0.047014523568478406) + | > loss_mel_ce: 3.4409079551696777 (3.5497782973087113) + | > loss: 0.8690293431282043 (0.8991982053674827) + | > current_lr: 5e-06 + | > step_time: 0.1793 (0.24430022867761478) + | > loader_time: 0.0109 (0.011820921088709952) + + + --> TIME: 2024-10-20 01:27:05 -- STEP: 24800/69059 -- GLOBAL_STEP: 24800 + | > loss_text_ce: 0.037717439234256744 (0.046993463342228155) + | > loss_mel_ce: 3.3462624549865723 (3.5493170288108926) + | > loss: 0.8459949493408203 (0.899077623188495) + | > current_lr: 5e-06 + | > step_time: 0.4456 (0.2443244781513368) + | > loader_time: 0.0128 (0.011820489514258577) + + + --> TIME: 2024-10-20 01:27:24 -- STEP: 24850/69059 -- GLOBAL_STEP: 24850 + | > loss_text_ce: 0.039031147956848145 (0.046972589453071326) + | > loss_mel_ce: 3.560288190841675 (3.5488732791186735) + | > loss: 0.8998298645019531 (0.8989614672991826) + | > current_lr: 5e-06 + | > step_time: 0.4044 (0.24436302739609655) + | > loader_time: 0.0125 (0.011820419578245206) + + + --> TIME: 2024-10-20 01:27:42 -- STEP: 24900/69059 -- GLOBAL_STEP: 24900 + | > loss_text_ce: 0.033907633274793625 (0.046951549397295164) + | > loss_mel_ce: 3.225081205368042 (3.548391847198739) + | > loss: 0.8147472143173218 (0.898835849304754) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.24434295638019302) + | > loader_time: 0.0103 (0.011819169348980984) + + + --> TIME: 2024-10-20 01:28:00 -- STEP: 24950/69059 -- GLOBAL_STEP: 24950 + | > loss_text_ce: 0.03151153773069382 (0.046930523315269056) + | > loss_mel_ce: 3.4074065685272217 (3.5478755661671926) + | > loss: 0.859729528427124 (0.898701522529483) + | > current_lr: 5e-06 + | > step_time: 0.2019 (0.24433938300681257) + | > loader_time: 0.0112 (0.011817734437380599) + + + --> TIME: 2024-10-20 01:28:19 -- STEP: 25000/69059 -- GLOBAL_STEP: 25000 + | > loss_text_ce: 0.032547879964113235 (0.046908972015827805) + | > loss_mel_ce: 3.3224329948425293 (3.5473535145568764) + | > loss: 0.8387452363967896 (0.8985656217980379) + | > current_lr: 5e-06 + | > step_time: 0.1559 (0.24433551833152772) + | > loader_time: 0.0105 (0.011816484508514382) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_25000.pth + + --> TIME: 2024-10-20 01:28:51 -- STEP: 25050/69059 -- GLOBAL_STEP: 25050 + | > loss_text_ce: 0.035511549562215805 (0.046887944148864484) + | > loss_mel_ce: 3.449117422103882 (3.5468238934404517) + | > loss: 0.8711572289466858 (0.8984279595473088) + | > current_lr: 5e-06 + | > step_time: 0.1855 (0.24433544034253576) + | > loader_time: 0.0102 (0.011816343528305915) + + + --> TIME: 2024-10-20 01:29:09 -- STEP: 25100/69059 -- GLOBAL_STEP: 25100 + | > loss_text_ce: 0.03841032087802887 (0.04686673669270657) + | > loss_mel_ce: 3.0939149856567383 (3.546411934269369) + | > loss: 0.7830813527107239 (0.8983196678912017) + | > current_lr: 5e-06 + | > step_time: 0.1713 (0.24431769352035218) + | > loader_time: 0.0104 (0.011815900489153592) + + + --> TIME: 2024-10-20 01:29:27 -- STEP: 25150/69059 -- GLOBAL_STEP: 25150 + | > loss_text_ce: 0.035080380737781525 (0.04684522316067158) + | > loss_mel_ce: 3.3138060569763184 (3.545954774302935) + | > loss: 0.837221622467041 (0.8981999995220247) + | > current_lr: 5e-06 + | > step_time: 0.2024 (0.244301723940946) + | > loader_time: 0.011 (0.011815225912135805) + + + --> TIME: 2024-10-20 01:29:45 -- STEP: 25200/69059 -- GLOBAL_STEP: 25200 + | > loss_text_ce: 0.03594411537051201 (0.046824606297064544) + | > loss_mel_ce: 3.0101258754730225 (3.5454973699838317) + | > loss: 0.7615175247192383 (0.898080494219348) + | > current_lr: 5e-06 + | > step_time: 0.1515 (0.24430791713888683) + | > loader_time: 0.0102 (0.011814994026744154) + + + --> TIME: 2024-10-20 01:30:04 -- STEP: 25250/69059 -- GLOBAL_STEP: 25250 + | > loss_text_ce: 0.03848646953701973 (0.046804587030617226) + | > loss_mel_ce: 3.1386406421661377 (3.5449890424048465) + | > loss: 0.7942817807197571 (0.8979484075130797) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24433205160764185) + | > loader_time: 0.0107 (0.01181497804245144) + + + --> TIME: 2024-10-20 01:30:24 -- STEP: 25300/69059 -- GLOBAL_STEP: 25300 + | > loss_text_ce: 0.03761610388755798 (0.04678391426558719) + | > loss_mel_ce: 3.308337450027466 (3.5444508281338276) + | > loss: 0.8364883661270142 (0.8978086857550694) + | > current_lr: 5e-06 + | > step_time: 0.2587 (0.24434774477020083) + | > loader_time: 0.0124 (0.011814777050093678) + + + --> TIME: 2024-10-20 01:30:41 -- STEP: 25350/69059 -- GLOBAL_STEP: 25350 + | > loss_text_ce: 0.03524254634976387 (0.046763058604646746) + | > loss_mel_ce: 3.0963709354400635 (3.5439030691575644) + | > loss: 0.7829033732414246 (0.8976665320927805) + | > current_lr: 5e-06 + | > step_time: 0.2035 (0.24433199516650017) + | > loader_time: 0.0139 (0.011814415073959058) + + + --> TIME: 2024-10-20 01:31:01 -- STEP: 25400/69059 -- GLOBAL_STEP: 25400 + | > loss_text_ce: 0.03693626448512077 (0.04674243896884292) + | > loss_mel_ce: 3.4555470943450928 (3.543466700221602) + | > loss: 0.8731208443641663 (0.8975522849573858) + | > current_lr: 5e-06 + | > step_time: 0.3432 (0.2443593901348865) + | > loader_time: 0.0131 (0.011814083968560504) + + + --> TIME: 2024-10-20 01:31:20 -- STEP: 25450/69059 -- GLOBAL_STEP: 25450 + | > loss_text_ce: 0.037234630435705185 (0.04672221992882088) + | > loss_mel_ce: 3.2188689708709717 (3.543063978774132) + | > loss: 0.81402587890625 (0.8974465498357479) + | > current_lr: 5e-06 + | > step_time: 0.1958 (0.24439112981785022) + | > loader_time: 0.0109 (0.011813915529981789) + + + --> TIME: 2024-10-20 01:31:39 -- STEP: 25500/69059 -- GLOBAL_STEP: 25500 + | > loss_text_ce: 0.03875826671719551 (0.04670152624674571) + | > loss_mel_ce: 3.3105764389038086 (3.542624281135252) + | > loss: 0.8373336791992188 (0.8973314520050492) + | > current_lr: 5e-06 + | > step_time: 0.3078 (0.244397123224595) + | > loader_time: 0.0116 (0.01181361368590708) + + + --> TIME: 2024-10-20 01:31:58 -- STEP: 25550/69059 -- GLOBAL_STEP: 25550 + | > loss_text_ce: 0.03555499389767647 (0.046681539712659564) + | > loss_mel_ce: 3.1388368606567383 (3.542148050972619) + | > loss: 0.793597936630249 (0.8972073978290627) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24441061967972907) + | > loader_time: 0.0109 (0.011813181804350889) + + + --> TIME: 2024-10-20 01:32:16 -- STEP: 25600/69059 -- GLOBAL_STEP: 25600 + | > loss_text_ce: 0.03455473110079765 (0.046660898985646695) + | > loss_mel_ce: 3.2650959491729736 (3.541669823862604) + | > loss: 0.8249126672744751 (0.8970826808689156) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.24439297595992684) + | > loader_time: 0.0106 (0.011812842665240146) + + + --> TIME: 2024-10-20 01:32:34 -- STEP: 25650/69059 -- GLOBAL_STEP: 25650 + | > loss_text_ce: 0.037849053740501404 (0.04663955594995731) + | > loss_mel_ce: 3.2823193073272705 (3.5411694531784574) + | > loss: 0.8300420641899109 (0.896952252436799) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.2443691011804354) + | > loader_time: 0.0103 (0.011812363618995689) + + + --> TIME: 2024-10-20 01:32:52 -- STEP: 25700/69059 -- GLOBAL_STEP: 25700 + | > loss_text_ce: 0.03415780887007713 (0.04661858450585119) + | > loss_mel_ce: 3.16129207611084 (3.540669890312825) + | > loss: 0.7988624572753906 (0.8968221188542438) + | > current_lr: 5e-06 + | > step_time: 0.1523 (0.2443597986540442) + | > loader_time: 0.0116 (0.011811554654562961) + + + --> TIME: 2024-10-20 01:33:11 -- STEP: 25750/69059 -- GLOBAL_STEP: 25750 + | > loss_text_ce: 0.03725298121571541 (0.046597393982184654) + | > loss_mel_ce: 3.048866033554077 (3.5402384865121856) + | > loss: 0.7715297341346741 (0.8967089702643232) + | > current_lr: 5e-06 + | > step_time: 0.2357 (0.244375046554121) + | > loader_time: 0.0112 (0.011811595453799327) + + + --> TIME: 2024-10-20 01:33:31 -- STEP: 25800/69059 -- GLOBAL_STEP: 25800 + | > loss_text_ce: 0.039402227848768234 (0.046577605614655154) + | > loss_mel_ce: 3.3484561443328857 (3.539754238368921) + | > loss: 0.8469645977020264 (0.8965829611309734) + | > current_lr: 5e-06 + | > step_time: 0.1646 (0.24441681040349858) + | > loader_time: 0.0104 (0.011811689175376575) + + + --> TIME: 2024-10-20 01:33:50 -- STEP: 25850/69059 -- GLOBAL_STEP: 25850 + | > loss_text_ce: 0.03566746786236763 (0.04655866306253504) + | > loss_mel_ce: 3.5500495433807373 (3.5392538670091533) + | > loss: 0.8964292407035828 (0.8964531326570634) + | > current_lr: 5e-06 + | > step_time: 0.2578 (0.2444315456375624) + | > loader_time: 0.0114 (0.011810803247374286) + + + --> TIME: 2024-10-20 01:34:08 -- STEP: 25900/69059 -- GLOBAL_STEP: 25900 + | > loss_text_ce: 0.03675856068730354 (0.04653865031898013) + | > loss_mel_ce: 3.06833815574646 (3.538862505023536) + | > loss: 0.7762742042541504 (0.8963502889710498) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.24442219206725307) + | > loader_time: 0.011 (0.011809999372047785) + + + --> TIME: 2024-10-20 01:34:27 -- STEP: 25950/69059 -- GLOBAL_STEP: 25950 + | > loss_text_ce: 0.035899385809898376 (0.04651832182843208) + | > loss_mel_ce: 3.3228564262390137 (3.5383913664183786) + | > loss: 0.8396889567375183 (0.896227422201564) + | > current_lr: 5e-06 + | > step_time: 0.1964 (0.24443350679842263) + | > loader_time: 0.0109 (0.011809249953268099) + + + --> TIME: 2024-10-20 01:34:46 -- STEP: 26000/69059 -- GLOBAL_STEP: 26000 + | > loss_text_ce: 0.03765217587351799 (0.04649747648663237) + | > loss_mel_ce: 3.3656697273254395 (3.538041598365849) + | > loss: 0.8508304953575134 (0.8961347688574052) + | > current_lr: 5e-06 + | > step_time: 0.1972 (0.24445839033677028) + | > loader_time: 0.0102 (0.011809126184536839) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_26000.pth + + --> TIME: 2024-10-20 01:35:18 -- STEP: 26050/69059 -- GLOBAL_STEP: 26050 + | > loss_text_ce: 0.04227607324719429 (0.046478055814663556) + | > loss_mel_ce: 3.498026132583618 (3.537536052540938) + | > loss: 0.885075569152832 (0.8960035272294403) + | > current_lr: 5e-06 + | > step_time: 0.3798 (0.2444567011871631) + | > loader_time: 0.0135 (0.011808682714451275) + + + --> TIME: 2024-10-20 01:35:36 -- STEP: 26100/69059 -- GLOBAL_STEP: 26100 + | > loss_text_ce: 0.034615810960531235 (0.04645905712940561) + | > loss_mel_ce: 3.318808078765869 (3.537119906149597) + | > loss: 0.8383559584617615 (0.8958947409569527) + | > current_lr: 5e-06 + | > step_time: 0.1629 (0.2444288760218127) + | > loader_time: 0.0107 (0.011807959308112697) + + + --> TIME: 2024-10-20 01:35:55 -- STEP: 26150/69059 -- GLOBAL_STEP: 26150 + | > loss_text_ce: 0.03711729124188423 (0.04643873450269711) + | > loss_mel_ce: 3.1580305099487305 (3.536648260767548) + | > loss: 0.7987869381904602 (0.8957717489626389) + | > current_lr: 5e-06 + | > step_time: 0.3233 (0.24443664631925624) + | > loader_time: 0.0121 (0.011807885352329794) + + + --> TIME: 2024-10-20 01:36:15 -- STEP: 26200/69059 -- GLOBAL_STEP: 26200 + | > loss_text_ce: 0.03752182424068451 (0.046419562153617644) + | > loss_mel_ce: 3.24858021736145 (3.536145342579317) + | > loss: 0.821525514125824 (0.8956412263271457) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.24449087999249233) + | > loader_time: 0.0105 (0.011808043017642166) + + + --> TIME: 2024-10-20 01:36:33 -- STEP: 26250/69059 -- GLOBAL_STEP: 26250 + | > loss_text_ce: 0.036676328629255295 (0.046400081073173366) + | > loss_mel_ce: 3.445787191390991 (3.5356604530788527) + | > loss: 0.8706158995628357 (0.8955151336806155) + | > current_lr: 5e-06 + | > step_time: 0.348 (0.24447700684865314) + | > loader_time: 0.0121 (0.011807812418256465) + + + --> TIME: 2024-10-20 01:36:51 -- STEP: 26300/69059 -- GLOBAL_STEP: 26300 + | > loss_text_ce: 0.033199138939380646 (0.04638067228025518) + | > loss_mel_ce: 3.4258832931518555 (3.5351605278366796) + | > loss: 0.8647705912590027 (0.8953853001703322) + | > current_lr: 5e-06 + | > step_time: 0.3796 (0.24446172433207697) + | > loader_time: 0.0138 (0.011807046281067568) + + + --> TIME: 2024-10-20 01:37:08 -- STEP: 26350/69059 -- GLOBAL_STEP: 26350 + | > loss_text_ce: 0.03515850380063057 (0.046361299109993245) + | > loss_mel_ce: 3.3918681144714355 (3.5346400508030014) + | > loss: 0.8567566275596619 (0.895250337614732) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.2444316990959124) + | > loader_time: 0.014 (0.011806968158743618) + + + --> TIME: 2024-10-20 01:37:27 -- STEP: 26400/69059 -- GLOBAL_STEP: 26400 + | > loss_text_ce: 0.039475638419389725 (0.046342447923679725) + | > loss_mel_ce: 3.3931691646575928 (3.534173320071256) + | > loss: 0.858161211013794 (0.8951289421274801) + | > current_lr: 5e-06 + | > step_time: 0.1566 (0.24441912452379863) + | > loader_time: 0.0104 (0.011806535720825174) + + + --> TIME: 2024-10-20 01:37:45 -- STEP: 26450/69059 -- GLOBAL_STEP: 26450 + | > loss_text_ce: 0.03408116102218628 (0.04632316564461507) + | > loss_mel_ce: 3.361359119415283 (3.533779804936421) + | > loss: 0.8488600850105286 (0.8950257427773534) + | > current_lr: 5e-06 + | > step_time: 0.1883 (0.24441325729420596) + | > loader_time: 0.012 (0.01180651890082258) + + + --> TIME: 2024-10-20 01:38:02 -- STEP: 26500/69059 -- GLOBAL_STEP: 26500 + | > loss_text_ce: 0.036950767040252686 (0.046304563690960886) + | > loss_mel_ce: 3.244955062866211 (3.533211596722865) + | > loss: 0.8204764723777771 (0.8948790402412409) + | > current_lr: 5e-06 + | > step_time: 0.1962 (0.24437582933677818) + | > loader_time: 0.011 (0.011805511249686165) + + + --> TIME: 2024-10-20 01:38:19 -- STEP: 26550/69059 -- GLOBAL_STEP: 26550 + | > loss_text_ce: 0.034236397594213486 (0.04628582684761659) + | > loss_mel_ce: 3.3737361431121826 (3.5327366212981266) + | > loss: 0.8519931435585022 (0.8947556121717508) + | > current_lr: 5e-06 + | > step_time: 0.1762 (0.24432222350169036) + | > loader_time: 0.0106 (0.011804183880937055) + + + --> TIME: 2024-10-20 01:38:38 -- STEP: 26600/69059 -- GLOBAL_STEP: 26600 + | > loss_text_ce: 0.037474535405635834 (0.04626668008345458) + | > loss_mel_ce: 3.3271403312683105 (3.5322462821723746) + | > loss: 0.8411537408828735 (0.8946282407036396) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.24433222221252612) + | > loader_time: 0.0111 (0.011803421956255899) + + + --> TIME: 2024-10-20 01:38:57 -- STEP: 26650/69059 -- GLOBAL_STEP: 26650 + | > loss_text_ce: 0.04059874266386032 (0.0462482078200097) + | > loss_mel_ce: 3.3782665729522705 (3.5318073993507437) + | > loss: 0.8547163009643555 (0.8945139019305884) + | > current_lr: 5e-06 + | > step_time: 0.4472 (0.24434881205854003) + | > loader_time: 0.0172 (0.011803010409142221) + + + --> TIME: 2024-10-20 01:39:14 -- STEP: 26700/69059 -- GLOBAL_STEP: 26700 + | > loss_text_ce: 0.03236020356416702 (0.04622854122480418) + | > loss_mel_ce: 3.4633586406707764 (3.531370410535212) + | > loss: 0.8739297389984131 (0.8943997380795992) + | > current_lr: 5e-06 + | > step_time: 0.2042 (0.24432828018281344) + | > loader_time: 0.011 (0.011802478086635839) + + + --> TIME: 2024-10-20 01:39:33 -- STEP: 26750/69059 -- GLOBAL_STEP: 26750 + | > loss_text_ce: 0.03595079109072685 (0.04621026044707981) + | > loss_mel_ce: 2.8161871433258057 (3.5308666918389076) + | > loss: 0.7130345106124878 (0.8942692382090555) + | > current_lr: 5e-06 + | > step_time: 0.2076 (0.2443297636976866) + | > loader_time: 0.0126 (0.011802507578769555) + + + --> TIME: 2024-10-20 01:39:50 -- STEP: 26800/69059 -- GLOBAL_STEP: 26800 + | > loss_text_ce: 0.036445505917072296 (0.046190777366508215) + | > loss_mel_ce: 3.363740921020508 (3.5303675440620945) + | > loss: 0.8500466346740723 (0.8941395804886493) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24429755084550203) + | > loader_time: 0.0105 (0.011801467555672353) + + + --> TIME: 2024-10-20 01:40:08 -- STEP: 26850/69059 -- GLOBAL_STEP: 26850 + | > loss_text_ce: 0.0354117825627327 (0.04617178043784145) + | > loss_mel_ce: 3.3545758724212646 (3.529902684124703) + | > loss: 0.8474969267845154 (0.894018616270086) + | > current_lr: 5e-06 + | > step_time: 0.2356 (0.24428749611923814) + | > loader_time: 0.0112 (0.011801065972397426) + + + --> TIME: 2024-10-20 01:40:28 -- STEP: 26900/69059 -- GLOBAL_STEP: 26900 + | > loss_text_ce: 0.03727754205465317 (0.04615223326093637) + | > loss_mel_ce: 3.238579034805298 (3.529496952849242) + | > loss: 0.8189641237258911 (0.8939122966585544) + | > current_lr: 5e-06 + | > step_time: 0.26 (0.24431645435914673) + | > loader_time: 0.0109 (0.01180064828865588) + + + --> TIME: 2024-10-20 01:40:48 -- STEP: 26950/69059 -- GLOBAL_STEP: 26950 + | > loss_text_ce: 0.03764024004340172 (0.0461342446694015) + | > loss_mel_ce: 3.179617166519165 (3.529098755840018) + | > loss: 0.804314374923706 (0.8938082502613699) + | > current_lr: 5e-06 + | > step_time: 0.1944 (0.24435582079560061) + | > loader_time: 0.0106 (0.011800098622663544) + + + --> TIME: 2024-10-20 01:41:06 -- STEP: 27000/69059 -- GLOBAL_STEP: 27000 + | > loss_text_ce: 0.037290167063474655 (0.04611551842797125) + | > loss_mel_ce: 2.944798707962036 (3.5286586732687697) + | > loss: 0.7455222010612488 (0.8936935480546063) + | > current_lr: 5e-06 + | > step_time: 0.1818 (0.24434268589372987) + | > loader_time: 0.0102 (0.011798964579900085) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_27000.pth + + --> TIME: 2024-10-20 01:41:41 -- STEP: 27050/69059 -- GLOBAL_STEP: 27050 + | > loss_text_ce: 0.03403019532561302 (0.046096733162479045) + | > loss_mel_ce: 2.9130215644836426 (3.528305525356645) + | > loss: 0.7367629408836365 (0.8936005647609938) + | > current_lr: 5e-06 + | > step_time: 0.1713 (0.24433127177620992) + | > loader_time: 0.0107 (0.011798587145073777) + + + --> TIME: 2024-10-20 01:41:58 -- STEP: 27100/69059 -- GLOBAL_STEP: 27100 + | > loss_text_ce: 0.033188220113515854 (0.04607723019536013) + | > loss_mel_ce: 3.1577670574188232 (3.5277763464705894) + | > loss: 0.797738790512085 (0.8934633943075618) + | > current_lr: 5e-06 + | > step_time: 0.2283 (0.24429682569750122) + | > loader_time: 0.012 (0.011798393268866711) + + + --> TIME: 2024-10-20 01:42:16 -- STEP: 27150/69059 -- GLOBAL_STEP: 27150 + | > loss_text_ce: 0.03428037464618683 (0.046058256582633045) + | > loss_mel_ce: 3.213595390319824 (3.5273472742744496) + | > loss: 0.8119689226150513 (0.8933513828469898) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.244270423033839) + | > loader_time: 0.0115 (0.011798044963436212) + + + --> TIME: 2024-10-20 01:42:33 -- STEP: 27200/69059 -- GLOBAL_STEP: 27200 + | > loss_text_ce: 0.03467954695224762 (0.04603949718574492) + | > loss_mel_ce: 3.0105209350585938 (3.5268498918939963) + | > loss: 0.7613001465797424 (0.8932223473949462) + | > current_lr: 5e-06 + | > step_time: 0.1631 (0.24423632932059905) + | > loader_time: 0.0105 (0.01179733757587038) + + + --> TIME: 2024-10-20 01:42:51 -- STEP: 27250/69059 -- GLOBAL_STEP: 27250 + | > loss_text_ce: 0.0356871522963047 (0.0460201996767192) + | > loss_mel_ce: 2.9017930030822754 (3.5264229928987763) + | > loss: 0.7343700528144836 (0.8931107982714237) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.2442214084852726) + | > loader_time: 0.0109 (0.011797203527678022) + + + --> TIME: 2024-10-20 01:43:11 -- STEP: 27300/69059 -- GLOBAL_STEP: 27300 + | > loss_text_ce: 0.03332789987325668 (0.046002010436312316) + | > loss_mel_ce: 3.181246757507324 (3.525962084213012) + | > loss: 0.8036436438560486 (0.8929910237907048) + | > current_lr: 5e-06 + | > step_time: 0.1573 (0.24424253035814333) + | > loader_time: 0.0103 (0.011797331497346068) + + + --> TIME: 2024-10-20 01:43:30 -- STEP: 27350/69059 -- GLOBAL_STEP: 27350 + | > loss_text_ce: 0.03998331353068352 (0.04598362865066062) + | > loss_mel_ce: 3.536264657974243 (3.5255362945762294) + | > loss: 0.8940619826316833 (0.8928799809376539) + | > current_lr: 5e-06 + | > step_time: 0.1846 (0.24426843980548588) + | > loader_time: 0.0123 (0.011797534630547037) + + + --> TIME: 2024-10-20 01:43:48 -- STEP: 27400/69059 -- GLOBAL_STEP: 27400 + | > loss_text_ce: 0.03587467968463898 (0.04596540256994802) + | > loss_mel_ce: 3.108600616455078 (3.525079127410895) + | > loss: 0.7861188054084778 (0.8927611326301179) + | > current_lr: 5e-06 + | > step_time: 0.1909 (0.24425106683786768) + | > loader_time: 0.0106 (0.011796852846215221) + + + --> TIME: 2024-10-20 01:44:07 -- STEP: 27450/69059 -- GLOBAL_STEP: 27450 + | > loss_text_ce: 0.036356713622808456 (0.045946977873941725) + | > loss_mel_ce: 3.14046311378479 (3.524644031672305) + | > loss: 0.7942049503326416 (0.8926477525251588) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.24427972653741611) + | > loader_time: 0.0105 (0.011796634696654696) + + + --> TIME: 2024-10-20 01:44:28 -- STEP: 27500/69059 -- GLOBAL_STEP: 27500 + | > loss_text_ce: 0.03354453667998314 (0.045929137844389054) + | > loss_mel_ce: 3.0448405742645264 (3.5242603882095955) + | > loss: 0.76959627866745 (0.8925473816524847) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.24434504967602816) + | > loader_time: 0.0113 (0.011797079779885011) + + + --> TIME: 2024-10-20 01:44:46 -- STEP: 27550/69059 -- GLOBAL_STEP: 27550 + | > loss_text_ce: 0.038515813648700714 (0.04591113631608473) + | > loss_mel_ce: 3.0500409603118896 (3.523868776302365) + | > loss: 0.7721391916275024 (0.8924449782925373) + | > current_lr: 5e-06 + | > step_time: 0.3537 (0.24433158114687717) + | > loader_time: 0.0104 (0.011796166849222873) + + + --> TIME: 2024-10-20 01:45:03 -- STEP: 27600/69059 -- GLOBAL_STEP: 27600 + | > loss_text_ce: 0.032848406583070755 (0.04589221162357076) + | > loss_mel_ce: 3.2051494121551514 (3.5234408083946733) + | > loss: 0.8094994425773621 (0.8923332551424049) + | > current_lr: 5e-06 + | > step_time: 0.1573 (0.24429160296916963) + | > loader_time: 0.01 (0.011794957121213256) + + + --> TIME: 2024-10-20 01:45:22 -- STEP: 27650/69059 -- GLOBAL_STEP: 27650 + | > loss_text_ce: 0.04293496534228325 (0.04587411982510871) + | > loss_mel_ce: 3.684786558151245 (3.5229825061656608) + | > loss: 0.9319303631782532 (0.8922141566328281) + | > current_lr: 5e-06 + | > step_time: 0.5565 (0.2442999906177762) + | > loader_time: 0.0147 (0.011794274995814159) + + + --> TIME: 2024-10-20 01:45:41 -- STEP: 27700/69059 -- GLOBAL_STEP: 27700 + | > loss_text_ce: 0.03681817278265953 (0.045856750886416525) + | > loss_mel_ce: 3.181522846221924 (3.522513875591188) + | > loss: 0.8045852780342102 (0.8920926567621604) + | > current_lr: 5e-06 + | > step_time: 0.3195 (0.24431910782514496) + | > loader_time: 0.0129 (0.011794140906970833) + + + --> TIME: 2024-10-20 01:46:01 -- STEP: 27750/69059 -- GLOBAL_STEP: 27750 + | > loss_text_ce: 0.03809744864702225 (0.0458391146995462) + | > loss_mel_ce: 3.488058567047119 (3.5221080765251567) + | > loss: 0.881538987159729 (0.891986797944919) + | > current_lr: 5e-06 + | > step_time: 0.4502 (0.24436526769345945) + | > loader_time: 0.0132 (0.011794239198839322) + + + --> TIME: 2024-10-20 01:46:21 -- STEP: 27800/69059 -- GLOBAL_STEP: 27800 + | > loss_text_ce: 0.0351828895509243 (0.04582049101550901) + | > loss_mel_ce: 3.096893072128296 (3.521729729072646) + | > loss: 0.7830190062522888 (0.8918875551566798) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.24440835710909728) + | > loader_time: 0.0111 (0.01179406652347645) + + + --> TIME: 2024-10-20 01:46:40 -- STEP: 27850/69059 -- GLOBAL_STEP: 27850 + | > loss_text_ce: 0.035060957074165344 (0.045803036536733725) + | > loss_mel_ce: 3.348159074783325 (3.5213375143003134) + | > loss: 0.8458049893379211 (0.8917851378523891) + | > current_lr: 5e-06 + | > step_time: 0.1791 (0.24442875984541176) + | > loader_time: 0.0112 (0.011793578283148972) + + + --> TIME: 2024-10-20 01:46:59 -- STEP: 27900/69059 -- GLOBAL_STEP: 27900 + | > loss_text_ce: 0.03765770420432091 (0.04578536537025236) + | > loss_mel_ce: 3.166886806488037 (3.520961053627786) + | > loss: 0.8011361360549927 (0.8916866048903443) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.2444389592618498) + | > loader_time: 0.0109 (0.011792966952033346) + + + --> TIME: 2024-10-20 01:47:18 -- STEP: 27950/69059 -- GLOBAL_STEP: 27950 + | > loss_text_ce: 0.037773728370666504 (0.04576821791664531) + | > loss_mel_ce: 3.5531184673309326 (3.5205478944420103) + | > loss: 0.8977230787277222 (0.8915790282272477) + | > current_lr: 5e-06 + | > step_time: 0.4506 (0.24445104401098808) + | > loader_time: 0.0126 (0.011792127365289713) + + + --> TIME: 2024-10-20 01:47:36 -- STEP: 28000/69059 -- GLOBAL_STEP: 28000 + | > loss_text_ce: 0.034786157310009 (0.045750904304507505) + | > loss_mel_ce: 3.3197784423828125 (3.5200835791996474) + | > loss: 0.8386411666870117 (0.8914586210144413) + | > current_lr: 5e-06 + | > step_time: 0.1745 (0.2444437967964581) + | > loader_time: 0.0125 (0.011791737837450824) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_28000.pth + + --> TIME: 2024-10-20 01:48:12 -- STEP: 28050/69059 -- GLOBAL_STEP: 28050 + | > loss_text_ce: 0.033022407442331314 (0.04573370087546506) + | > loss_mel_ce: 3.286764621734619 (3.519618201196293) + | > loss: 0.829946756362915 (0.8913379756652092) + | > current_lr: 5e-06 + | > step_time: 0.3055 (0.24446203794496368) + | > loader_time: 0.0117 (0.011792280125745458) + + + --> TIME: 2024-10-20 01:48:32 -- STEP: 28100/69059 -- GLOBAL_STEP: 28100 + | > loss_text_ce: 0.039565570652484894 (0.045715997624558934) + | > loss_mel_ce: 3.496619462966919 (3.5191763057488066) + | > loss: 0.8840462565422058 (0.8912230759880292) + | > current_lr: 5e-06 + | > step_time: 0.251 (0.24450166939840623) + | > loader_time: 0.0117 (0.011792509021283874) + + + --> TIME: 2024-10-20 01:48:50 -- STEP: 28150/69059 -- GLOBAL_STEP: 28150 + | > loss_text_ce: 0.03692911937832832 (0.0456977146795783) + | > loss_mel_ce: 3.589390754699707 (3.518790236883004) + | > loss: 0.9065799713134766 (0.8911219880381862) + | > current_lr: 5e-06 + | > step_time: 0.3219 (0.24447605581944196) + | > loader_time: 0.0123 (0.011791925836837504) + + + --> TIME: 2024-10-20 01:49:09 -- STEP: 28200/69059 -- GLOBAL_STEP: 28200 + | > loss_text_ce: 0.037072401493787766 (0.045680272605679584) + | > loss_mel_ce: 3.049961566925049 (3.518412864487215) + | > loss: 0.771758496761322 (0.891023284420899) + | > current_lr: 5e-06 + | > step_time: 0.1628 (0.24448560034975092) + | > loader_time: 0.0107 (0.011791821033396599) + + + --> TIME: 2024-10-20 01:49:27 -- STEP: 28250/69059 -- GLOBAL_STEP: 28250 + | > loss_text_ce: 0.036412809044122696 (0.045662823966976206) + | > loss_mel_ce: 3.0076940059661865 (3.517915215956419) + | > loss: 0.7610266804695129 (0.890894510134131) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.24448093309655652) + | > loader_time: 0.011 (0.011792137492019497) + + + --> TIME: 2024-10-20 01:49:46 -- STEP: 28300/69059 -- GLOBAL_STEP: 28300 + | > loss_text_ce: 0.038218241184949875 (0.045646322620802156) + | > loss_mel_ce: 3.3470242023468018 (3.5174949638666546) + | > loss: 0.8463106155395508 (0.8907853217828395) + | > current_lr: 5e-06 + | > step_time: 0.3562 (0.2445010610267046) + | > loader_time: 0.0125 (0.01179130002382364) + + + --> TIME: 2024-10-20 01:50:06 -- STEP: 28350/69059 -- GLOBAL_STEP: 28350 + | > loss_text_ce: 0.03136983886361122 (0.04562891473674997) + | > loss_mel_ce: 3.2629237174987793 (3.5172020314652173) + | > loss: 0.8235734105110168 (0.8907077367175398) + | > current_lr: 5e-06 + | > step_time: 0.1962 (0.24454004447506428) + | > loader_time: 0.0104 (0.011791041700507788) + + + --> TIME: 2024-10-20 01:50:26 -- STEP: 28400/69059 -- GLOBAL_STEP: 28400 + | > loss_text_ce: 0.03804530203342438 (0.04561163565183286) + | > loss_mel_ce: 3.236248254776001 (3.51683003862978) + | > loss: 0.8185734152793884 (0.8906104187394527) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.2445609378730747) + | > loader_time: 0.0103 (0.011790696046721748) + + + --> TIME: 2024-10-20 01:50:44 -- STEP: 28450/69059 -- GLOBAL_STEP: 28450 + | > loss_text_ce: 0.03378725051879883 (0.04559374861929865) + | > loss_mel_ce: 3.079009532928467 (3.5163465435936425) + | > loss: 0.7781991958618164 (0.8904850732190742) + | > current_lr: 5e-06 + | > step_time: 0.1874 (0.24455217359774142) + | > loader_time: 0.0119 (0.011790289954267593) + + + --> TIME: 2024-10-20 01:51:02 -- STEP: 28500/69059 -- GLOBAL_STEP: 28500 + | > loss_text_ce: 0.03846750780940056 (0.04557649968956638) + | > loss_mel_ce: 3.208834171295166 (3.515887651309625) + | > loss: 0.8118253946304321 (0.8903660379104442) + | > current_lr: 5e-06 + | > step_time: 0.3758 (0.24454197604196112) + | > loader_time: 0.013 (0.011789523944520094) + + + --> TIME: 2024-10-20 01:51:22 -- STEP: 28550/69059 -- GLOBAL_STEP: 28550 + | > loss_text_ce: 0.03264649212360382 (0.04555965923986369) + | > loss_mel_ce: 3.020209550857544 (3.5154203225351184) + | > loss: 0.7632139921188354 (0.8902449956019575) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.2445924549303205) + | > loader_time: 0.0105 (0.0117895147303566) + + + --> TIME: 2024-10-20 01:51:42 -- STEP: 28600/69059 -- GLOBAL_STEP: 28600 + | > loss_text_ce: 0.035574059933423996 (0.04554207208416566) + | > loss_mel_ce: 3.409332275390625 (3.5150269754699823) + | > loss: 0.8612265586853027 (0.8901422620549063) + | > current_lr: 5e-06 + | > step_time: 0.3081 (0.2446196846695213) + | > loader_time: 0.0122 (0.011789265695985362) + + + --> TIME: 2024-10-20 01:52:01 -- STEP: 28650/69059 -- GLOBAL_STEP: 28650 + | > loss_text_ce: 0.035603612661361694 (0.04552494121302581) + | > loss_mel_ce: 3.0229671001434326 (3.5145581598015494) + | > loss: 0.7646426558494568 (0.890020775420503) + | > current_lr: 5e-06 + | > step_time: 0.1708 (0.2446219177479103) + | > loader_time: 0.01 (0.01178837524034588) + + + --> TIME: 2024-10-20 01:52:19 -- STEP: 28700/69059 -- GLOBAL_STEP: 28700 + | > loss_text_ce: 0.03700442612171173 (0.045507625068313906) + | > loss_mel_ce: 3.2325241565704346 (3.5141074195556037) + | > loss: 0.8173821568489075 (0.889903761320828) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.2446172699197244) + | > loader_time: 0.0113 (0.011787916110367705) + + + --> TIME: 2024-10-20 01:52:38 -- STEP: 28750/69059 -- GLOBAL_STEP: 28750 + | > loss_text_ce: 0.03451118245720863 (0.04549012638667348) + | > loss_mel_ce: 3.4936575889587402 (3.5136675535782453) + | > loss: 0.8820421695709229 (0.889789420156893) + | > current_lr: 5e-06 + | > step_time: 0.2316 (0.2446388844448587) + | > loader_time: 0.0116 (0.011787929211492104) + + + --> TIME: 2024-10-20 01:52:56 -- STEP: 28800/69059 -- GLOBAL_STEP: 28800 + | > loss_text_ce: 0.03551353141665459 (0.04547283405399252) + | > loss_mel_ce: 3.1156773567199707 (3.5131888596382335) + | > loss: 0.787797749042511 (0.8896654235964843) + | > current_lr: 5e-06 + | > step_time: 0.2868 (0.24461573195954164) + | > loader_time: 0.0115 (0.011787030465073036) + + + --> TIME: 2024-10-20 01:53:14 -- STEP: 28850/69059 -- GLOBAL_STEP: 28850 + | > loss_text_ce: 0.0364156998693943 (0.045455382539918064) + | > loss_mel_ce: 3.158724308013916 (3.5127653523664883) + | > loss: 0.7987850308418274 (0.8895551839011169) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.24461542622055504) + | > loader_time: 0.0109 (0.01178648132178769) + + + --> TIME: 2024-10-20 01:53:33 -- STEP: 28900/69059 -- GLOBAL_STEP: 28900 + | > loss_text_ce: 0.03521905094385147 (0.04543853685530507) + | > loss_mel_ce: 3.1167213916778564 (3.5122485242582866) + | > loss: 0.78798508644104 (0.8894217654577996) + | > current_lr: 5e-06 + | > step_time: 0.3561 (0.24461399107243362) + | > loader_time: 0.013 (0.011786192428694445) + + + --> TIME: 2024-10-20 01:53:51 -- STEP: 28950/69059 -- GLOBAL_STEP: 28950 + | > loss_text_ce: 0.03656696900725365 (0.04542154098743491) + | > loss_mel_ce: 3.206223964691162 (3.5118284312678054) + | > loss: 0.8106977343559265 (0.8893124932390416) + | > current_lr: 5e-06 + | > step_time: 0.185 (0.24460691699092252) + | > loader_time: 0.0101 (0.011785273074280187) + + + --> TIME: 2024-10-20 01:54:10 -- STEP: 29000/69059 -- GLOBAL_STEP: 29000 + | > loss_text_ce: 0.03417172282934189 (0.0454044652750779) + | > loss_mel_ce: 3.33624005317688 (3.511473655207397) + | > loss: 0.8426029682159424 (0.8892195303008469) + | > current_lr: 5e-06 + | > step_time: 0.1514 (0.24462077322499506) + | > loader_time: 0.0104 (0.011784407278587058) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_29000.pth + + --> TIME: 2024-10-20 01:54:46 -- STEP: 29050/69059 -- GLOBAL_STEP: 29050 + | > loss_text_ce: 0.03427661582827568 (0.045387340479124766) + | > loss_mel_ce: 3.2733542919158936 (3.5110490242585395) + | > loss: 0.826907753944397 (0.8891090913688865) + | > current_lr: 5e-06 + | > step_time: 0.2063 (0.2446121634304421) + | > loader_time: 0.0107 (0.011784319401608052) + + + --> TIME: 2024-10-20 01:55:05 -- STEP: 29100/69059 -- GLOBAL_STEP: 29100 + | > loss_text_ce: 0.03616256266832352 (0.04537079648694009) + | > loss_mel_ce: 3.346897602081299 (3.510638479907891) + | > loss: 0.8457650542259216 (0.8890023192855495) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.24459842595857442) + | > loader_time: 0.0107 (0.01178439624530751) + + + --> TIME: 2024-10-20 01:55:24 -- STEP: 29150/69059 -- GLOBAL_STEP: 29150 + | > loss_text_ce: 0.03120277263224125 (0.04535455424453589) + | > loss_mel_ce: 3.026526689529419 (3.5102838230460147) + | > loss: 0.7644323706626892 (0.8889095945096542) + | > current_lr: 5e-06 + | > step_time: 0.1714 (0.24462853776038612) + | > loader_time: 0.0102 (0.01178412251775095) + + + --> TIME: 2024-10-20 01:55:44 -- STEP: 29200/69059 -- GLOBAL_STEP: 29200 + | > loss_text_ce: 0.03537352383136749 (0.04533840569954562) + | > loss_mel_ce: 3.3367395401000977 (3.5098704939509022) + | > loss: 0.8430282473564148 (0.8888022251018918) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.24466365543130325) + | > loader_time: 0.013 (0.01178436427083732) + + + --> TIME: 2024-10-20 01:56:04 -- STEP: 29250/69059 -- GLOBAL_STEP: 29250 + | > loss_text_ce: 0.034986503422260284 (0.04532288635624008) + | > loss_mel_ce: 3.278975248336792 (3.5094650552374613) + | > loss: 0.8284904360771179 (0.8886969855830196) + | > current_lr: 5e-06 + | > step_time: 0.2051 (0.24469479677412245) + | > loader_time: 0.0111 (0.011784829490205142) + + + --> TIME: 2024-10-20 01:56:22 -- STEP: 29300/69059 -- GLOBAL_STEP: 29300 + | > loss_text_ce: 0.03209352493286133 (0.045305810750306284) + | > loss_mel_ce: 3.2424964904785156 (3.509093077736089) + | > loss: 0.8186475038528442 (0.8885997223101373) + | > current_lr: 5e-06 + | > step_time: 0.1635 (0.24467738705690403) + | > loader_time: 0.0104 (0.011784419159026664) + + + --> TIME: 2024-10-20 01:56:41 -- STEP: 29350/69059 -- GLOBAL_STEP: 29350 + | > loss_text_ce: 0.03475160151720047 (0.0452885216302547) + | > loss_mel_ce: 3.3316760063171387 (3.5086679753574774) + | > loss: 0.8416069149971008 (0.8884891244378158) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.2446790365790996) + | > loader_time: 0.0105 (0.011785080526309668) + + + --> TIME: 2024-10-20 01:56:59 -- STEP: 29400/69059 -- GLOBAL_STEP: 29400 + | > loss_text_ce: 0.03674042969942093 (0.045272384080338866) + | > loss_mel_ce: 3.5076510906219482 (3.5082736994944392) + | > loss: 0.8860979080200195 (0.8883865210859949) + | > current_lr: 5e-06 + | > step_time: 0.2034 (0.24467411407808057) + | > loader_time: 0.0119 (0.011784977507429038) + + + --> TIME: 2024-10-20 01:57:20 -- STEP: 29450/69059 -- GLOBAL_STEP: 29450 + | > loss_text_ce: 0.037681903690099716 (0.04525679918856736) + | > loss_mel_ce: 3.1971049308776855 (3.507922664401075) + | > loss: 0.8086966872215271 (0.8882948660911237) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.24472295955405374) + | > loader_time: 0.0107 (0.01178531539055607) + + + --> TIME: 2024-10-20 01:57:39 -- STEP: 29500/69059 -- GLOBAL_STEP: 29500 + | > loss_text_ce: 0.03425944223999977 (0.045240692606606146) + | > loss_mel_ce: 3.282301187515259 (3.5074340720984827) + | > loss: 0.8291401863098145 (0.8881686913744873) + | > current_lr: 5e-06 + | > step_time: 0.3464 (0.2447395798957954) + | > loader_time: 0.0142 (0.011785732729960276) + + + --> TIME: 2024-10-20 01:57:58 -- STEP: 29550/69059 -- GLOBAL_STEP: 29550 + | > loss_text_ce: 0.03598373383283615 (0.045224238871809416) + | > loss_mel_ce: 3.3652188777923584 (3.507080461999076) + | > loss: 0.850300669670105 (0.8880761754129537) + | > current_lr: 5e-06 + | > step_time: 0.5169 (0.24477247639154218) + | > loader_time: 0.0197 (0.011786213284216538) + + + --> TIME: 2024-10-20 01:58:18 -- STEP: 29600/69059 -- GLOBAL_STEP: 29600 + | > loss_text_ce: 0.03413555026054382 (0.04520814130743454) + | > loss_mel_ce: 3.1279215812683105 (3.5066723006802567) + | > loss: 0.7905142903327942 (0.8879701106914792) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.24480028687296687) + | > loader_time: 0.0104 (0.011786064062569573) + + + --> TIME: 2024-10-20 01:58:36 -- STEP: 29650/69059 -- GLOBAL_STEP: 29650 + | > loss_text_ce: 0.03404141962528229 (0.045191910783076994) + | > loss_mel_ce: 3.0873045921325684 (3.5062697012388213) + | > loss: 0.7803364992141724 (0.8878654032030953) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.244795005470263) + | > loader_time: 0.0115 (0.011785663858835031) + + + --> TIME: 2024-10-20 01:58:55 -- STEP: 29700/69059 -- GLOBAL_STEP: 29700 + | > loss_text_ce: 0.03549661859869957 (0.04517657815375254) + | > loss_mel_ce: 3.415888786315918 (3.5058218003844503) + | > loss: 0.8628463745117188 (0.8877495948333125) + | > current_lr: 5e-06 + | > step_time: 0.2534 (0.2448006052039689) + | > loader_time: 0.0143 (0.011785365515686427) + + + --> TIME: 2024-10-20 01:59:13 -- STEP: 29750/69059 -- GLOBAL_STEP: 29750 + | > loss_text_ce: 0.035764746367931366 (0.04516026966142296) + | > loss_mel_ce: 3.2594261169433594 (3.5053946127931543) + | > loss: 0.8237977027893066 (0.8876387208169243) + | > current_lr: 5e-06 + | > step_time: 0.1794 (0.2447714123926243) + | > loader_time: 0.0107 (0.011784801138549272) + + + --> TIME: 2024-10-20 01:59:32 -- STEP: 29800/69059 -- GLOBAL_STEP: 29800 + | > loss_text_ce: 0.036643870174884796 (0.0451443871316318) + | > loss_mel_ce: 3.003354549407959 (3.5049545443297965) + | > loss: 0.7599996328353882 (0.8875247330713587) + | > current_lr: 5e-06 + | > step_time: 0.1646 (0.24478887035542687) + | > loader_time: 0.011 (0.01178462553344315) + + + --> TIME: 2024-10-20 01:59:52 -- STEP: 29850/69059 -- GLOBAL_STEP: 29850 + | > loss_text_ce: 0.03404949977993965 (0.04512862237847373) + | > loss_mel_ce: 3.2481746673583984 (3.504591412672039) + | > loss: 0.8205560445785522 (0.8874300089573336) + | > current_lr: 5e-06 + | > step_time: 0.1851 (0.24483831439185982) + | > loader_time: 0.0111 (0.01178488844802829) + + + --> TIME: 2024-10-20 02:00:09 -- STEP: 29900/69059 -- GLOBAL_STEP: 29900 + | > loss_text_ce: 0.03546180576086044 (0.045112077609688765) + | > loss_mel_ce: 3.1972997188568115 (3.5041620668200677) + | > loss: 0.8081904053688049 (0.8873185362963368) + | > current_lr: 5e-06 + | > step_time: 0.2465 (0.2447884151369433) + | > loader_time: 0.0119 (0.011783879895672731) + + + --> TIME: 2024-10-20 02:00:28 -- STEP: 29950/69059 -- GLOBAL_STEP: 29950 + | > loss_text_ce: 0.03588574752211571 (0.04509578039767042) + | > loss_mel_ce: 3.417429208755493 (3.503829727045476) + | > loss: 0.863328754901886 (0.8872313770507527) + | > current_lr: 5e-06 + | > step_time: 0.3807 (0.24481831907230944) + | > loader_time: 0.0116 (0.011783548523866256) + + + --> TIME: 2024-10-20 02:00:48 -- STEP: 30000/69059 -- GLOBAL_STEP: 30000 + | > loss_text_ce: 0.034426312893629074 (0.04507963251434259) + | > loss_mel_ce: 2.952937126159668 (3.5034090855598383) + | > loss: 0.7468408346176147 (0.887122179704904) + | > current_lr: 5e-06 + | > step_time: 0.1777 (0.24483677944342294) + | > loader_time: 0.0105 (0.011783287088076255) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_30000.pth + + --> TIME: 2024-10-20 02:01:21 -- STEP: 30050/69059 -- GLOBAL_STEP: 30050 + | > loss_text_ce: 0.03598814085125923 (0.045063746284674806) + | > loss_mel_ce: 3.3417654037475586 (3.503037394453794) + | > loss: 0.844438374042511 (0.8870252853661721) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.2448562327795933) + | > loader_time: 0.0107 (0.011782860986008205) + + + --> TIME: 2024-10-20 02:01:40 -- STEP: 30100/69059 -- GLOBAL_STEP: 30100 + | > loss_text_ce: 0.037405867129564285 (0.04504770984074891) + | > loss_mel_ce: 3.1357216835021973 (3.5026735049862334) + | > loss: 0.7932819128036499 (0.8869303038864828) + | > current_lr: 5e-06 + | > step_time: 0.2763 (0.24485692585821564) + | > loader_time: 0.0123 (0.011782629474056897) + + + --> TIME: 2024-10-20 02:01:57 -- STEP: 30150/69059 -- GLOBAL_STEP: 30150 + | > loss_text_ce: 0.03706567734479904 (0.045032055873553474) + | > loss_mel_ce: 3.3497464656829834 (3.5022024809819876) + | > loss: 0.846703052520752 (0.8868086343954249) + | > current_lr: 5e-06 + | > step_time: 0.5902 (0.24483695810707054) + | > loader_time: 0.0152 (0.011782186090649639) + + + --> TIME: 2024-10-20 02:02:18 -- STEP: 30200/69059 -- GLOBAL_STEP: 30200 + | > loss_text_ce: 0.03308218717575073 (0.04501635222017313) + | > loss_mel_ce: 3.21126651763916 (3.5018629251331617) + | > loss: 0.8110871911048889 (0.8867198195165351) + | > current_lr: 5e-06 + | > step_time: 0.3352 (0.24488008746248208) + | > loader_time: 0.0126 (0.0117831799368195) + + + --> TIME: 2024-10-20 02:02:36 -- STEP: 30250/69059 -- GLOBAL_STEP: 30250 + | > loss_text_ce: 0.031102150678634644 (0.04500062994018559) + | > loss_mel_ce: 2.8915724754333496 (3.5014234983586014) + | > loss: 0.7306686639785767 (0.8866060322493556) + | > current_lr: 5e-06 + | > step_time: 0.1769 (0.24487683762794685) + | > loader_time: 0.0112 (0.011783471312404646) + + + --> TIME: 2024-10-20 02:02:55 -- STEP: 30300/69059 -- GLOBAL_STEP: 30300 + | > loss_text_ce: 0.032119665294885635 (0.044984680477148556) + | > loss_mel_ce: 3.1694717407226562 (3.501074010924531) + | > loss: 0.8003978729248047 (0.886514673026481) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.24490025402295706) + | > loader_time: 0.0108 (0.011783523897920093) + + + --> TIME: 2024-10-20 02:03:13 -- STEP: 30350/69059 -- GLOBAL_STEP: 30350 + | > loss_text_ce: 0.039986394345760345 (0.04496908683992633) + | > loss_mel_ce: 3.4091410636901855 (3.5006577085623762) + | > loss: 0.862281858921051 (0.8864066990301671) + | > current_lr: 5e-06 + | > step_time: 0.1791 (0.24488760540473598) + | > loader_time: 0.0107 (0.011783018819189917) + + + --> TIME: 2024-10-20 02:03:31 -- STEP: 30400/69059 -- GLOBAL_STEP: 30400 + | > loss_text_ce: 0.03820883482694626 (0.04495384565585187) + | > loss_mel_ce: 3.5375821590423584 (3.500292365809798) + | > loss: 0.8939477205276489 (0.8863115530480675) + | > current_lr: 5e-06 + | > step_time: 0.3246 (0.24486592012016398) + | > loader_time: 0.0129 (0.011782385416721023) + + + --> TIME: 2024-10-20 02:03:50 -- STEP: 30450/69059 -- GLOBAL_STEP: 30450 + | > loss_text_ce: 0.03604719042778015 (0.04493790631976173) + | > loss_mel_ce: 3.1962599754333496 (3.499855547154863) + | > loss: 0.808076798915863 (0.8861983635488202) + | > current_lr: 5e-06 + | > step_time: 0.2358 (0.24487425303028526) + | > loader_time: 0.0116 (0.011782214167865804) + + + --> TIME: 2024-10-20 02:04:09 -- STEP: 30500/69059 -- GLOBAL_STEP: 30500 + | > loss_text_ce: 0.03607571870088577 (0.04492254095396294) + | > loss_mel_ce: 3.4904608726501465 (3.499521938081641) + | > loss: 0.8816341757774353 (0.886111119938678) + | > current_lr: 5e-06 + | > step_time: 0.3115 (0.2448930072940764) + | > loader_time: 0.013 (0.011782202884799125) + + + --> TIME: 2024-10-20 02:04:29 -- STEP: 30550/69059 -- GLOBAL_STEP: 30550 + | > loss_text_ce: 0.03437049314379692 (0.0449074819587085) + | > loss_mel_ce: 3.1971118450164795 (3.4991786107376956) + | > loss: 0.8078705668449402 (0.8860215233566323) + | > current_lr: 5e-06 + | > step_time: 0.1941 (0.24491463023589988) + | > loader_time: 0.0118 (0.011782028593135135) + + + --> TIME: 2024-10-20 02:04:47 -- STEP: 30600/69059 -- GLOBAL_STEP: 30600 + | > loss_text_ce: 0.03632824495434761 (0.044892545704200204) + | > loss_mel_ce: 3.3013131618499756 (3.498919165555162) + | > loss: 0.8344103693962097 (0.8859529279980779) + | > current_lr: 5e-06 + | > step_time: 0.3085 (0.24491464198025223) + | > loader_time: 0.0124 (0.011781973581688058) + + + --> TIME: 2024-10-20 02:05:06 -- STEP: 30650/69059 -- GLOBAL_STEP: 30650 + | > loss_text_ce: 0.03262685239315033 (0.04487703209253686) + | > loss_mel_ce: 3.127880334854126 (3.4985728547390225) + | > loss: 0.7901268005371094 (0.885862471890099) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24492628649047304) + | > loader_time: 0.0114 (0.011782158350672244) + + + --> TIME: 2024-10-20 02:05:25 -- STEP: 30700/69059 -- GLOBAL_STEP: 30700 + | > loss_text_ce: 0.03602229431271553 (0.04486133601285922) + | > loss_mel_ce: 3.1229593753814697 (3.498192889519539) + | > loss: 0.7897453904151917 (0.885763556566999) + | > current_lr: 5e-06 + | > step_time: 0.3351 (0.2449207424030242) + | > loader_time: 0.0123 (0.011781712541362732) + + + --> TIME: 2024-10-20 02:05:44 -- STEP: 30750/69059 -- GLOBAL_STEP: 30750 + | > loss_text_ce: 0.035154398530721664 (0.04484598823721566) + | > loss_mel_ce: 3.1633827686309814 (3.4978334019125894) + | > loss: 0.7996342778205872 (0.8856698477210063) + | > current_lr: 5e-06 + | > step_time: 0.2342 (0.24492630733707088) + | > loader_time: 0.0118 (0.011781384181200954) + + + --> TIME: 2024-10-20 02:06:02 -- STEP: 30800/69059 -- GLOBAL_STEP: 30800 + | > loss_text_ce: 0.035885684192180634 (0.044830186073184425) + | > loss_mel_ce: 3.2760603427886963 (3.497427479764077) + | > loss: 0.827986478805542 (0.8855644166430865) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.24493193504098174) + | > loader_time: 0.0104 (0.011781539034533791) + + + --> TIME: 2024-10-20 02:06:21 -- STEP: 30850/69059 -- GLOBAL_STEP: 30850 + | > loss_text_ce: 0.033760394901037216 (0.04481472605238608) + | > loss_mel_ce: 3.382814645767212 (3.4970503596703018) + | > loss: 0.8541437387466431 (0.8854662716176169) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.24492734317068535) + | > loader_time: 0.0108 (0.011780792132768659) + + + --> TIME: 2024-10-20 02:06:41 -- STEP: 30900/69059 -- GLOBAL_STEP: 30900 + | > loss_text_ce: 0.03469676151871681 (0.04479954924283717) + | > loss_mel_ce: 3.5624217987060547 (3.4966864968040547) + | > loss: 0.8992796540260315 (0.8853715116981545) + | > current_lr: 5e-06 + | > step_time: 0.1968 (0.24496950177313054) + | > loader_time: 0.0112 (0.011780483660960257) + + + --> TIME: 2024-10-20 02:06:59 -- STEP: 30950/69059 -- GLOBAL_STEP: 30950 + | > loss_text_ce: 0.034215666353702545 (0.04478393546129556) + | > loss_mel_ce: 3.1394481658935547 (3.4963399967893003) + | > loss: 0.7934159636497498 (0.8852809832438894) + | > current_lr: 5e-06 + | > step_time: 0.2335 (0.24495257794375566) + | > loader_time: 0.0119 (0.011779833155803015) + + + --> TIME: 2024-10-20 02:07:17 -- STEP: 31000/69059 -- GLOBAL_STEP: 31000 + | > loss_text_ce: 0.03499191999435425 (0.0447682230971873) + | > loss_mel_ce: 3.1059412956237793 (3.49590460568089) + | > loss: 0.7852333188056946 (0.8851682073723881) + | > current_lr: 5e-06 + | > step_time: 0.1663 (0.24493494994409623) + | > loader_time: 0.0123 (0.011779135250276117) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_31000.pth + + --> TIME: 2024-10-20 02:07:56 -- STEP: 31050/69059 -- GLOBAL_STEP: 31050 + | > loss_text_ce: 0.038438353687524796 (0.044753489104098) + | > loss_mel_ce: 3.0486209392547607 (3.4956206775441006) + | > loss: 0.7717648148536682 (0.8850935418383122) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.24496111738509027) + | > loader_time: 0.0109 (0.011779396960124859) + + + --> TIME: 2024-10-20 02:08:15 -- STEP: 31100/69059 -- GLOBAL_STEP: 31100 + | > loss_text_ce: 0.032861899584531784 (0.04473802098363535) + | > loss_mel_ce: 3.2287228107452393 (3.495222096305178) + | > loss: 0.8153961896896362 (0.8849900294998446) + | > current_lr: 5e-06 + | > step_time: 0.2608 (0.24497026215795537) + | > loader_time: 0.0142 (0.01177938399974171) + + + --> TIME: 2024-10-20 02:08:33 -- STEP: 31150/69059 -- GLOBAL_STEP: 31150 + | > loss_text_ce: 0.032089393585920334 (0.04472303200985506) + | > loss_mel_ce: 3.3553855419158936 (3.494841932347262) + | > loss: 0.8468687534332275 (0.8848912412664859) + | > current_lr: 5e-06 + | > step_time: 0.1775 (0.24496731159583715) + | > loader_time: 0.0112 (0.011779731823965578) + + + --> TIME: 2024-10-20 02:08:51 -- STEP: 31200/69059 -- GLOBAL_STEP: 31200 + | > loss_text_ce: 0.03126150369644165 (0.04470735873740446) + | > loss_mel_ce: 3.236140727996826 (3.4944365859184443) + | > loss: 0.8168505430221558 (0.884785986334467) + | > current_lr: 5e-06 + | > step_time: 0.1713 (0.24494072613043663) + | > loader_time: 0.0122 (0.011779160270324102) + + + --> TIME: 2024-10-20 02:09:10 -- STEP: 31250/69059 -- GLOBAL_STEP: 31250 + | > loss_text_ce: 0.03817354887723923 (0.0446918964313864) + | > loss_mel_ce: 3.218632936477661 (3.4940845395889215) + | > loss: 0.8142015933990479 (0.8846941091747279) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.24495825096130372) + | > loader_time: 0.0105 (0.011779031188964825) + + + --> TIME: 2024-10-20 02:09:28 -- STEP: 31300/69059 -- GLOBAL_STEP: 31300 + | > loss_text_ce: 0.03782614320516586 (0.04467742348899135) + | > loss_mel_ce: 3.1697888374328613 (3.493775633607803) + | > loss: 0.8019037246704102 (0.8846132644449175) + | > current_lr: 5e-06 + | > step_time: 0.4731 (0.24495620204236943) + | > loader_time: 0.0145 (0.011778720400203894) + + + --> TIME: 2024-10-20 02:09:47 -- STEP: 31350/69059 -- GLOBAL_STEP: 31350 + | > loss_text_ce: 0.03357832133769989 (0.044662209473300706) + | > loss_mel_ce: 3.477963447570801 (3.493433383739349) + | > loss: 0.8778854608535767 (0.8845238984790901) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.2449660127432913) + | > loader_time: 0.0102 (0.011778254052668649) + + + --> TIME: 2024-10-20 02:10:07 -- STEP: 31400/69059 -- GLOBAL_STEP: 31400 + | > loss_text_ce: 0.03506435826420784 (0.04464714460644365) + | > loss_mel_ce: 3.236093044281006 (3.4930986561213264) + | > loss: 0.8177893757820129 (0.8844364503519545) + | > current_lr: 5e-06 + | > step_time: 0.358 (0.24500870091899946) + | > loader_time: 0.0128 (0.011778249201501233) + + + --> TIME: 2024-10-20 02:10:27 -- STEP: 31450/69059 -- GLOBAL_STEP: 31450 + | > loss_text_ce: 0.03605075553059578 (0.044632658960211756) + | > loss_mel_ce: 3.029366970062256 (3.4927918996416643) + | > loss: 0.7663544416427612 (0.8843561398205957) + | > current_lr: 5e-06 + | > step_time: 0.1725 (0.24502666433210024) + | > loader_time: 0.0109 (0.011778222540414182) + + + --> TIME: 2024-10-20 02:10:47 -- STEP: 31500/69059 -- GLOBAL_STEP: 31500 + | > loss_text_ce: 0.03414466232061386 (0.044618050942108675) + | > loss_mel_ce: 3.276463747024536 (3.4924599074560434) + | > loss: 0.827652096748352 (0.884269489767059) + | > current_lr: 5e-06 + | > step_time: 0.1655 (0.24506178070250012) + | > loader_time: 0.0118 (0.01177840268422685) + + + --> TIME: 2024-10-20 02:11:04 -- STEP: 31550/69059 -- GLOBAL_STEP: 31550 + | > loss_text_ce: 0.03657256439328194 (0.04460348429161275) + | > loss_mel_ce: 3.314359426498413 (3.4920462610377685) + | > loss: 0.8377329707145691 (0.8841624364913355) + | > current_lr: 5e-06 + | > step_time: 0.2809 (0.24502356386411397) + | > loader_time: 0.0116 (0.011777758734350897) + + + --> TIME: 2024-10-20 02:11:23 -- STEP: 31600/69059 -- GLOBAL_STEP: 31600 + | > loss_text_ce: 0.034325938671827316 (0.044588930084358214) + | > loss_mel_ce: 3.544881820678711 (3.4916250188893847) + | > loss: 0.894801914691925 (0.8840534874042374) + | > current_lr: 5e-06 + | > step_time: 0.347 (0.24503991115696822) + | > loader_time: 0.0125 (0.011777546617049188) + + + --> TIME: 2024-10-20 02:11:41 -- STEP: 31650/69059 -- GLOBAL_STEP: 31650 + | > loss_text_ce: 0.033408649265766144 (0.04457457034697081) + | > loss_mel_ce: 3.2396628856658936 (3.491240141952954) + | > loss: 0.8182678818702698 (0.8839536782369413) + | > current_lr: 5e-06 + | > step_time: 0.2115 (0.2450321933782496) + | > loader_time: 0.0111 (0.01177665249431301) + + + --> TIME: 2024-10-20 02:12:01 -- STEP: 31700/69059 -- GLOBAL_STEP: 31700 + | > loss_text_ce: 0.03784355893731117 (0.04456043852093602) + | > loss_mel_ce: 3.2256147861480713 (3.4909135417005555) + | > loss: 0.8158645629882812 (0.883868495212744) + | > current_lr: 5e-06 + | > step_time: 0.1908 (0.24506324235573176) + | > loader_time: 0.0115 (0.01177660385516914) + + + --> TIME: 2024-10-20 02:12:20 -- STEP: 31750/69059 -- GLOBAL_STEP: 31750 + | > loss_text_ce: 0.03488265350461006 (0.04454577142534052) + | > loss_mel_ce: 3.2817139625549316 (3.490583321729038) + | > loss: 0.8291491270065308 (0.8837822734464806) + | > current_lr: 5e-06 + | > step_time: 0.3666 (0.24507147688377562) + | > loader_time: 0.0173 (0.011776321290984848) + + + --> TIME: 2024-10-20 02:12:37 -- STEP: 31800/69059 -- GLOBAL_STEP: 31800 + | > loss_text_ce: 0.03474927321076393 (0.04453103613008814) + | > loss_mel_ce: 3.0882792472839355 (3.4901732607682483) + | > loss: 0.7807571291923523 (0.8836760743740215) + | > current_lr: 5e-06 + | > step_time: 0.181 (0.2450379772456187) + | > loader_time: 0.0112 (0.0117752254833965) + + + --> TIME: 2024-10-20 02:12:56 -- STEP: 31850/69059 -- GLOBAL_STEP: 31850 + | > loss_text_ce: 0.03319529816508293 (0.044517932540592275) + | > loss_mel_ce: 3.012120008468628 (3.4897371975780405) + | > loss: 0.7613288164138794 (0.8835637826728666) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.2450527191536393) + | > loader_time: 0.0107 (0.01177454241778074) + + + --> TIME: 2024-10-20 02:13:15 -- STEP: 31900/69059 -- GLOBAL_STEP: 31900 + | > loss_text_ce: 0.03566969186067581 (0.04450369609747554) + | > loss_mel_ce: 3.547006845474243 (3.489336477222855) + | > loss: 0.895669162273407 (0.8834600434706872) + | > current_lr: 5e-06 + | > step_time: 0.3234 (0.2450509988327385) + | > loader_time: 0.0119 (0.011774168619915218) + + + --> TIME: 2024-10-20 02:13:34 -- STEP: 31950/69059 -- GLOBAL_STEP: 31950 + | > loss_text_ce: 0.035702142864465714 (0.04448868196084505) + | > loss_mel_ce: 3.2156951427459717 (3.4890217854346033) + | > loss: 0.8128493428230286 (0.8833776169838102) + | > current_lr: 5e-06 + | > step_time: 0.281 (0.2450598377577016) + | > loader_time: 0.0118 (0.0117737013409395) + + + --> TIME: 2024-10-20 02:13:54 -- STEP: 32000/69059 -- GLOBAL_STEP: 32000 + | > loss_text_ce: 0.03615016117691994 (0.04447476159199134) + | > loss_mel_ce: 3.502716302871704 (3.4886622140407497) + | > loss: 0.8847166299819946 (0.8832842440418894) + | > current_lr: 5e-06 + | > step_time: 0.2743 (0.2450948524698615) + | > loader_time: 0.0126 (0.011773305140435678) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_32000.pth + + --> TIME: 2024-10-20 02:14:26 -- STEP: 32050/69059 -- GLOBAL_STEP: 32050 + | > loss_text_ce: 0.0384429469704628 (0.04445979292976682) + | > loss_mel_ce: 3.246241807937622 (3.488262071713636) + | > loss: 0.8211711645126343 (0.8831804662877791) + | > current_lr: 5e-06 + | > step_time: 0.2574 (0.2451009421489912) + | > loader_time: 0.0116 (0.011773228994955679) + + + --> TIME: 2024-10-20 02:14:44 -- STEP: 32100/69059 -- GLOBAL_STEP: 32100 + | > loss_text_ce: 0.03462541848421097 (0.044445699248752975) + | > loss_mel_ce: 2.9293372631073 (3.4878675483691666) + | > loss: 0.7409906983375549 (0.8830783120364039) + | > current_lr: 5e-06 + | > step_time: 0.1722 (0.24509133942030673) + | > loader_time: 0.0109 (0.0117724299430847) + + + --> TIME: 2024-10-20 02:15:04 -- STEP: 32150/69059 -- GLOBAL_STEP: 32150 + | > loss_text_ce: 0.03633904829621315 (0.044432139263646156) + | > loss_mel_ce: 3.513302803039551 (3.4875049153679387) + | > loss: 0.8874104619026184 (0.8829842637894123) + | > current_lr: 5e-06 + | > step_time: 0.2257 (0.24513125242378628) + | > loader_time: 0.0115 (0.011772693490166297) + + + --> TIME: 2024-10-20 02:15:22 -- STEP: 32200/69059 -- GLOBAL_STEP: 32200 + | > loss_text_ce: 0.03291618451476097 (0.04441755161915684) + | > loss_mel_ce: 3.098045587539673 (3.4871611105062885) + | > loss: 0.7827404141426086 (0.8828946656599543) + | > current_lr: 5e-06 + | > step_time: 0.1735 (0.24511243153803097) + | > loader_time: 0.011 (0.011772251292045054) + + + --> TIME: 2024-10-20 02:15:40 -- STEP: 32250/69059 -- GLOBAL_STEP: 32250 + | > loss_text_ce: 0.030715277418494225 (0.0444031099070065) + | > loss_mel_ce: 3.2683963775634766 (3.486804251471225) + | > loss: 0.8247779011726379 (0.8828018404679701) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.245118974515634) + | > loader_time: 0.0106 (0.011772253679674707) + + + --> TIME: 2024-10-20 02:16:00 -- STEP: 32300/69059 -- GLOBAL_STEP: 32300 + | > loss_text_ce: 0.03672955557703972 (0.044388607800779536) + | > loss_mel_ce: 3.2265021800994873 (3.4864741894630393) + | > loss: 0.8158079385757446 (0.882715699440185) + | > current_lr: 5e-06 + | > step_time: 0.1818 (0.24513233731774722) + | > loader_time: 0.0114 (0.011772423211265995) + + + --> TIME: 2024-10-20 02:16:19 -- STEP: 32350/69059 -- GLOBAL_STEP: 32350 + | > loss_text_ce: 0.03586163371801376 (0.04437492320362984) + | > loss_mel_ce: 3.4527230262756348 (3.4861311439236986) + | > loss: 0.8721461892127991 (0.8826265169037545) + | > current_lr: 5e-06 + | > step_time: 0.3432 (0.24516485799409138) + | > loader_time: 0.0133 (0.011772907255606067) + + + --> TIME: 2024-10-20 02:16:38 -- STEP: 32400/69059 -- GLOBAL_STEP: 32400 + | > loss_text_ce: 0.034509990364313126 (0.04436099313854888) + | > loss_mel_ce: 3.1053922176361084 (3.4857430215455807) + | > loss: 0.784975528717041 (0.8825260037846031) + | > current_lr: 5e-06 + | > step_time: 0.2073 (0.24515501898747905) + | > loader_time: 0.0105 (0.011772506936096834) + + + --> TIME: 2024-10-20 02:16:56 -- STEP: 32450/69059 -- GLOBAL_STEP: 32450 + | > loss_text_ce: 0.036832600831985474 (0.044346770952485765) + | > loss_mel_ce: 3.3771841526031494 (3.4854692628901556) + | > loss: 0.8535041809082031 (0.8824540085711717) + | > current_lr: 5e-06 + | > step_time: 0.2354 (0.2451640406668829) + | > loader_time: 0.0115 (0.011772176987952169) + + + --> TIME: 2024-10-20 02:17:16 -- STEP: 32500/69059 -- GLOBAL_STEP: 32500 + | > loss_text_ce: 0.03148721158504486 (0.04433294350871666) + | > loss_mel_ce: 3.3087363243103027 (3.485136745900368) + | > loss: 0.8350558876991272 (0.8823674224670113) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.24519030356040367) + | > loader_time: 0.0109 (0.01177265300017135) + + + --> TIME: 2024-10-20 02:17:34 -- STEP: 32550/69059 -- GLOBAL_STEP: 32550 + | > loss_text_ce: 0.03694342449307442 (0.044318975675421854) + | > loss_mel_ce: 3.426853656768799 (3.4847484906734163) + | > loss: 0.865949273109436 (0.8822668667011725) + | > current_lr: 5e-06 + | > step_time: 0.2654 (0.24516978006758447) + | > loader_time: 0.0114 (0.011772319036328464) + + + --> TIME: 2024-10-20 02:17:52 -- STEP: 32600/69059 -- GLOBAL_STEP: 32600 + | > loss_text_ce: 0.03811957687139511 (0.044304314881871656) + | > loss_mel_ce: 3.186596155166626 (3.4843793201080797) + | > loss: 0.8061789274215698 (0.8821709088624616) + | > current_lr: 5e-06 + | > step_time: 0.3106 (0.24516318315377264) + | > loader_time: 0.012 (0.011771752827006597) + + + --> TIME: 2024-10-20 02:18:09 -- STEP: 32650/69059 -- GLOBAL_STEP: 32650 + | > loss_text_ce: 0.03718456253409386 (0.04429010971758798) + | > loss_mel_ce: 3.5462687015533447 (3.484023255279558) + | > loss: 0.8958632946014404 (0.8820783413661379) + | > current_lr: 5e-06 + | > step_time: 0.2239 (0.24513553504739385) + | > loader_time: 0.0114 (0.011771140032851478) + + + --> TIME: 2024-10-20 02:18:30 -- STEP: 32700/69059 -- GLOBAL_STEP: 32700 + | > loss_text_ce: 0.03734651207923889 (0.04427660721778544) + | > loss_mel_ce: 3.145642042160034 (3.483672499190041) + | > loss: 0.7957471609115601 (0.881987276717063) + | > current_lr: 5e-06 + | > step_time: 0.1707 (0.24517668591362257) + | > loader_time: 0.0105 (0.01177151799566518) + + + --> TIME: 2024-10-20 02:18:47 -- STEP: 32750/69059 -- GLOBAL_STEP: 32750 + | > loss_text_ce: 0.033740535378456116 (0.044262677633523415) + | > loss_mel_ce: 3.184645414352417 (3.4832544635525124) + | > loss: 0.804596483707428 (0.8818792854119799) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.2451444410950173) + | > loader_time: 0.0104 (0.011770707094032329) + + + --> TIME: 2024-10-20 02:19:06 -- STEP: 32800/69059 -- GLOBAL_STEP: 32800 + | > loss_text_ce: 0.0358632430434227 (0.04424903399935667) + | > loss_mel_ce: 3.0268361568450928 (3.482862866847974) + | > loss: 0.7656748294830322 (0.8817779753266307) + | > current_lr: 5e-06 + | > step_time: 0.2739 (0.24515701094778572) + | > loader_time: 0.0109 (0.011770513428420538) + + + --> TIME: 2024-10-20 02:19:24 -- STEP: 32850/69059 -- GLOBAL_STEP: 32850 + | > loss_text_ce: 0.03371081128716469 (0.04423508614233633) + | > loss_mel_ce: 3.326920747756958 (3.482513492140045) + | > loss: 0.8401578664779663 (0.8816871446870047) + | > current_lr: 5e-06 + | > step_time: 0.2022 (0.24514353978579448) + | > loader_time: 0.0113 (0.011770917886649801) + + + --> TIME: 2024-10-20 02:19:42 -- STEP: 32900/69059 -- GLOBAL_STEP: 32900 + | > loss_text_ce: 0.03364865109324455 (0.044221099278688544) + | > loss_mel_ce: 3.254774808883667 (3.4821723519052714) + | > loss: 0.822105884552002 (0.8815983629099862) + | > current_lr: 5e-06 + | > step_time: 0.1508 (0.2451270674354762) + | > loader_time: 0.01 (0.011770775629756643) + + + --> TIME: 2024-10-20 02:20:00 -- STEP: 32950/69059 -- GLOBAL_STEP: 32950 + | > loss_text_ce: 0.03275545686483383 (0.044207308781933824) + | > loss_mel_ce: 3.1888513565063477 (3.4817934879221877) + | > loss: 0.8054016828536987 (0.8815001992842855) + | > current_lr: 5e-06 + | > step_time: 0.2063 (0.24511846110383007) + | > loader_time: 0.0117 (0.011770543248953678) + + + --> TIME: 2024-10-20 02:20:19 -- STEP: 33000/69059 -- GLOBAL_STEP: 33000 + | > loss_text_ce: 0.03605666384100914 (0.04419351073851181) + | > loss_mel_ce: 3.289144515991211 (3.4814629909992156) + | > loss: 0.8313003182411194 (0.8814141255418456) + | > current_lr: 5e-06 + | > step_time: 0.1833 (0.2451404881838596) + | > loader_time: 0.0115 (0.01177046249852034) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_33000.pth + + --> TIME: 2024-10-20 02:20:54 -- STEP: 33050/69059 -- GLOBAL_STEP: 33050 + | > loss_text_ce: 0.03485865145921707 (0.04417994601835184) + | > loss_mel_ce: 3.0818114280700684 (3.4811565075338997) + | > loss: 0.7791675329208374 (0.8813341134957932) + | > current_lr: 5e-06 + | > step_time: 0.2763 (0.24516935533185946) + | > loader_time: 0.012 (0.01177134940916397) + + + --> TIME: 2024-10-20 02:21:12 -- STEP: 33100/69059 -- GLOBAL_STEP: 33100 + | > loss_text_ce: 0.034970078617334366 (0.04416583234731247) + | > loss_mel_ce: 3.2131800651550293 (3.480878601693669) + | > loss: 0.8120375275611877 (0.8812611086130859) + | > current_lr: 5e-06 + | > step_time: 0.1729 (0.2451731444125449) + | > loader_time: 0.0108 (0.011771294732108197) + + + --> TIME: 2024-10-20 02:21:32 -- STEP: 33150/69059 -- GLOBAL_STEP: 33150 + | > loss_text_ce: 0.03395512327551842 (0.044152110230801254) + | > loss_mel_ce: 3.37088680267334 (3.480504169384992) + | > loss: 0.8512104749679565 (0.8811640700082071) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.2451892689737978) + | > loader_time: 0.0108 (0.011771236861273513) + + + --> TIME: 2024-10-20 02:21:49 -- STEP: 33200/69059 -- GLOBAL_STEP: 33200 + | > loss_text_ce: 0.03430097922682762 (0.044138347737895) + | > loss_mel_ce: 3.11609148979187 (3.480079569701683) + | > loss: 0.7875981330871582 (0.8810544794672223) + | > current_lr: 5e-06 + | > step_time: 0.1821 (0.24515696539218165) + | > loader_time: 0.011 (0.01177070600440701) + + + --> TIME: 2024-10-20 02:22:07 -- STEP: 33250/69059 -- GLOBAL_STEP: 33250 + | > loss_text_ce: 0.03858698531985283 (0.0441239598218659) + | > loss_mel_ce: 3.1400563716888428 (3.479718388550258) + | > loss: 0.7946608662605286 (0.8809605871985728) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.245138175720559) + | > loader_time: 0.0114 (0.011770539441503058) + + + --> TIME: 2024-10-20 02:22:27 -- STEP: 33300/69059 -- GLOBAL_STEP: 33300 + | > loss_text_ce: 0.03139382600784302 (0.04411070995491448) + | > loss_mel_ce: 3.2167465686798096 (3.4793909726128507) + | > loss: 0.812035083770752 (0.8808754207493663) + | > current_lr: 5e-06 + | > step_time: 0.1559 (0.24516831086562557) + | > loader_time: 0.0108 (0.011771032638377948) + + + --> TIME: 2024-10-20 02:22:46 -- STEP: 33350/69059 -- GLOBAL_STEP: 33350 + | > loss_text_ce: 0.034485284239053726 (0.04409764598692007) + | > loss_mel_ce: 3.5976297855377197 (3.4790557870621748) + | > loss: 0.908028781414032 (0.8807883583730843) + | > current_lr: 5e-06 + | > step_time: 0.3131 (0.24519775547187725) + | > loader_time: 0.0131 (0.011770994795494687) + + + --> TIME: 2024-10-20 02:23:04 -- STEP: 33400/69059 -- GLOBAL_STEP: 33400 + | > loss_text_ce: 0.033115535974502563 (0.04408478614366687) + | > loss_mel_ce: 3.3066344261169434 (3.478710931025573) + | > loss: 0.8349375128746033 (0.8806989294040702) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.24517214073392443) + | > loader_time: 0.0108 (0.011770363610661652) + + + --> TIME: 2024-10-20 02:23:22 -- STEP: 33450/69059 -- GLOBAL_STEP: 33450 + | > loss_text_ce: 0.03256823867559433 (0.044070886381427) + | > loss_mel_ce: 3.1016929149627686 (3.478313373963388) + | > loss: 0.7835652828216553 (0.8805960652004977) + | > current_lr: 5e-06 + | > step_time: 0.1723 (0.24516020255238485) + | > loader_time: 0.0104 (0.011769785567427589) + + + --> TIME: 2024-10-20 02:23:41 -- STEP: 33500/69059 -- GLOBAL_STEP: 33500 + | > loss_text_ce: 0.034467946738004684 (0.044057927015223514) + | > loss_mel_ce: 3.0323615074157715 (3.4779507064463435) + | > loss: 0.7667073607444763 (0.880502158478125) + | > current_lr: 5e-06 + | > step_time: 0.1648 (0.24517358488823052) + | > loader_time: 0.0113 (0.011769444010150948) + + + --> TIME: 2024-10-20 02:24:00 -- STEP: 33550/69059 -- GLOBAL_STEP: 33550 + | > loss_text_ce: 0.03661913797259331 (0.044044329734011047) + | > loss_mel_ce: 2.980675220489502 (3.47762876770002) + | > loss: 0.7543236017227173 (0.8804182744701291) + | > current_lr: 5e-06 + | > step_time: 0.1951 (0.24519122626138337) + | > loader_time: 0.0114 (0.011769435594049884) + + + --> TIME: 2024-10-20 02:24:18 -- STEP: 33600/69059 -- GLOBAL_STEP: 33600 + | > loss_text_ce: 0.0353393480181694 (0.0440310618050196) + | > loss_mel_ce: 2.876204252243042 (3.4772135864056235) + | > loss: 0.727885901927948 (0.8803111621666523) + | > current_lr: 5e-06 + | > step_time: 0.1654 (0.24517287047136396) + | > loader_time: 0.011 (0.011769086888858235) + + + --> TIME: 2024-10-20 02:24:35 -- STEP: 33650/69059 -- GLOBAL_STEP: 33650 + | > loss_text_ce: 0.035550087690353394 (0.044017271863858384) + | > loss_mel_ce: 2.978003740310669 (3.476865637653113) + | > loss: 0.7533884644508362 (0.880220727486561) + | > current_lr: 5e-06 + | > step_time: 0.1722 (0.24515333238717882) + | > loader_time: 0.0104 (0.011768468910659387) + + + --> TIME: 2024-10-20 02:24:55 -- STEP: 33700/69059 -- GLOBAL_STEP: 33700 + | > loss_text_ce: 0.0358889102935791 (0.04400432594114967) + | > loss_mel_ce: 3.4821062088012695 (3.4765503094104293) + | > loss: 0.8794987797737122 (0.8801386589448604) + | > current_lr: 5e-06 + | > step_time: 0.3008 (0.24517866818062983) + | > loader_time: 0.0131 (0.01176850536103063) + + + --> TIME: 2024-10-20 02:25:13 -- STEP: 33750/69059 -- GLOBAL_STEP: 33750 + | > loss_text_ce: 0.0346541628241539 (0.0439911202090205) + | > loss_mel_ce: 3.188594102859497 (3.476241080785675) + | > loss: 0.8058120608329773 (0.8800580503534392) + | > current_lr: 5e-06 + | > step_time: 0.1938 (0.24516297751532662) + | > loader_time: 0.011 (0.011767867060060836) + + + --> TIME: 2024-10-20 02:25:32 -- STEP: 33800/69059 -- GLOBAL_STEP: 33800 + | > loss_text_ce: 0.03425871953368187 (0.043978257260874856) + | > loss_mel_ce: 3.1651880741119385 (3.4758749067148456) + | > loss: 0.7998616695404053 (0.8799632911022602) + | > current_lr: 5e-06 + | > step_time: 0.1907 (0.24517650065337412) + | > loader_time: 0.0108 (0.011767603913707828) + + + --> TIME: 2024-10-20 02:25:51 -- STEP: 33850/69059 -- GLOBAL_STEP: 33850 + | > loss_text_ce: 0.036515314131975174 (0.043965496437753306) + | > loss_mel_ce: 3.2422971725463867 (3.475538711449238) + | > loss: 0.8197031021118164 (0.8798760520795604) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.24518390766098722) + | > loader_time: 0.0104 (0.011767149408247461) + + + --> TIME: 2024-10-20 02:26:09 -- STEP: 33900/69059 -- GLOBAL_STEP: 33900 + | > loss_text_ce: 0.03257344290614128 (0.043952212009062346) + | > loss_mel_ce: 3.5920369625091553 (3.4751928179383564) + | > loss: 0.906152606010437 (0.8797862575908681) + | > current_lr: 5e-06 + | > step_time: 0.3367 (0.24518585221957317) + | > loader_time: 0.0128 (0.011766773381416053) + + + --> TIME: 2024-10-20 02:26:28 -- STEP: 33950/69059 -- GLOBAL_STEP: 33950 + | > loss_text_ce: 0.03502573072910309 (0.04393916273739215) + | > loss_mel_ce: 3.0673139095306396 (3.474836062549309) + | > loss: 0.7755849361419678 (0.8796938064312549) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24518270591572916) + | > loader_time: 0.0111 (0.011766253697503453) + + + --> TIME: 2024-10-20 02:26:46 -- STEP: 34000/69059 -- GLOBAL_STEP: 34000 + | > loss_text_ce: 0.03531873971223831 (0.043925686270162786) + | > loss_mel_ce: 3.038177967071533 (3.474469630465781) + | > loss: 0.7683742046356201 (0.8795988292904463) + | > current_lr: 5e-06 + | > step_time: 0.1558 (0.2451794778879951) + | > loader_time: 0.01 (0.01176574925114125) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_34000.pth + + --> TIME: 2024-10-20 02:27:21 -- STEP: 34050/69059 -- GLOBAL_STEP: 34050 + | > loss_text_ce: 0.03598666936159134 (0.0439130699619781) + | > loss_mel_ce: 2.8877336978912354 (3.4741504952000724) + | > loss: 0.7309300899505615 (0.8795158913954765) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.24521204227217844) + | > loader_time: 0.0111 (0.011765834742530661) + + + --> TIME: 2024-10-20 02:27:40 -- STEP: 34100/69059 -- GLOBAL_STEP: 34100 + | > loss_text_ce: 0.036004647612571716 (0.04390017275368002) + | > loss_mel_ce: 2.9696266651153564 (3.4738003895149823) + | > loss: 0.75140780210495 (0.8794251406682203) + | > current_lr: 5e-06 + | > step_time: 0.2056 (0.2452205911269985) + | > loader_time: 0.0113 (0.011766004667254125) + + + --> TIME: 2024-10-20 02:27:59 -- STEP: 34150/69059 -- GLOBAL_STEP: 34150 + | > loss_text_ce: 0.03355991840362549 (0.043887438379544334) + | > loss_mel_ce: 3.321439743041992 (3.4734739569305075) + | > loss: 0.838749885559082 (0.8793403489226653) + | > current_lr: 5e-06 + | > step_time: 0.2592 (0.24522117581932995) + | > loader_time: 0.0121 (0.011765624028173747) + + + --> TIME: 2024-10-20 02:28:16 -- STEP: 34200/69059 -- GLOBAL_STEP: 34200 + | > loss_text_ce: 0.03375403955578804 (0.04387364099967709) + | > loss_mel_ce: 3.254340887069702 (3.473051862716667) + | > loss: 0.8220237493515015 (0.8792313760170464) + | > current_lr: 5e-06 + | > step_time: 0.2078 (0.24518840806526052) + | > loader_time: 0.0108 (0.011765071300038111) + + + --> TIME: 2024-10-20 02:28:34 -- STEP: 34250/69059 -- GLOBAL_STEP: 34250 + | > loss_text_ce: 0.03232341632246971 (0.0438609493036012) + | > loss_mel_ce: 3.569108724594116 (3.472813552856437) + | > loss: 0.9003580212593079 (0.8791686256262511) + | > current_lr: 5e-06 + | > step_time: 0.2237 (0.2451816299675155) + | > loader_time: 0.0142 (0.01176497428086552) + + + --> TIME: 2024-10-20 02:28:50 -- STEP: 34300/69059 -- GLOBAL_STEP: 34300 + | > loss_text_ce: 0.03351771831512451 (0.04384774351166683) + | > loss_mel_ce: 3.4373714923858643 (3.472452247879587) + | > loss: 0.8677222728729248 (0.8790749979279829) + | > current_lr: 5e-06 + | > step_time: 0.4615 (0.24513876187905626) + | > loader_time: 0.0151 (0.011764458907936468) + + + --> TIME: 2024-10-20 02:29:09 -- STEP: 34350/69059 -- GLOBAL_STEP: 34350 + | > loss_text_ce: 0.03617506101727486 (0.043835406109549536) + | > loss_mel_ce: 2.863980770111084 (3.47209661857182) + | > loss: 0.7250389456748962 (0.8789830062500429) + | > current_lr: 5e-06 + | > step_time: 0.1953 (0.2451414669097945) + | > loader_time: 0.0112 (0.011764376305770315) + + + --> TIME: 2024-10-20 02:29:26 -- STEP: 34400/69059 -- GLOBAL_STEP: 34400 + | > loss_text_ce: 0.03701498359441757 (0.043822008461074015) + | > loss_mel_ce: 3.163358211517334 (3.471802315677311) + | > loss: 0.8000932931900024 (0.8789060811195958) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24511185601700183) + | > loader_time: 0.0104 (0.01176457037759381) + + + --> TIME: 2024-10-20 02:29:45 -- STEP: 34450/69059 -- GLOBAL_STEP: 34450 + | > loss_text_ce: 0.032282568514347076 (0.043809646787648905) + | > loss_mel_ce: 3.115156412124634 (3.471475408489012) + | > loss: 0.7868597507476807 (0.8788212639067446) + | > current_lr: 5e-06 + | > step_time: 0.2108 (0.24510835950018187) + | > loader_time: 0.0114 (0.01176429063384522) + + + --> TIME: 2024-10-20 02:30:04 -- STEP: 34500/69059 -- GLOBAL_STEP: 34500 + | > loss_text_ce: 0.036459486931562424 (0.04379702434278467) + | > loss_mel_ce: 3.14070987701416 (3.4711411551116056) + | > loss: 0.7942923307418823 (0.8787345449527104) + | > current_lr: 5e-06 + | > step_time: 0.3315 (0.24512885356986006) + | > loader_time: 0.0125 (0.011764051008915548) + + + --> TIME: 2024-10-20 02:30:23 -- STEP: 34550/69059 -- GLOBAL_STEP: 34550 + | > loss_text_ce: 0.03991199657320976 (0.04378480622676953) + | > loss_mel_ce: 3.35436749458313 (3.4707661807174683) + | > loss: 0.8485698699951172 (0.8786377468312699) + | > current_lr: 5e-06 + | > step_time: 0.2226 (0.24513896479102879) + | > loader_time: 0.0113 (0.011763817915523094) + + + --> TIME: 2024-10-20 02:30:42 -- STEP: 34600/69059 -- GLOBAL_STEP: 34600 + | > loss_text_ce: 0.03537372872233391 (0.043772323046288586) + | > loss_mel_ce: 3.042226791381836 (3.470473306282397) + | > loss: 0.7694001197814941 (0.8785614074258444) + | > current_lr: 5e-06 + | > step_time: 0.1643 (0.24515144177254913) + | > loader_time: 0.0103 (0.011763372262778302) + + + --> TIME: 2024-10-20 02:31:01 -- STEP: 34650/69059 -- GLOBAL_STEP: 34650 + | > loss_text_ce: 0.03595171496272087 (0.043758912658665466) + | > loss_mel_ce: 3.2487776279449463 (3.470104408697638) + | > loss: 0.8211823105812073 (0.8784658304334202) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.2451532714053838) + | > loader_time: 0.0104 (0.011762961336716594) + + + --> TIME: 2024-10-20 02:31:20 -- STEP: 34700/69059 -- GLOBAL_STEP: 34700 + | > loss_text_ce: 0.03586031496524811 (0.0437459988534578) + | > loss_mel_ce: 3.3294787406921387 (3.4698082750331314) + | > loss: 0.8413347601890564 (0.8783885685665802) + | > current_lr: 5e-06 + | > step_time: 0.2984 (0.24516894626342597) + | > loader_time: 0.0131 (0.011763170019693946) + + + --> TIME: 2024-10-20 02:31:37 -- STEP: 34750/69059 -- GLOBAL_STEP: 34750 + | > loss_text_ce: 0.03381480276584625 (0.04373412492088693) + | > loss_mel_ce: 3.1860744953155518 (3.4693786278525596) + | > loss: 0.8049723505973816 (0.8782781882869248) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24514642922655283) + | > loader_time: 0.0102 (0.011762672787947613) + + + --> TIME: 2024-10-20 02:31:57 -- STEP: 34800/69059 -- GLOBAL_STEP: 34800 + | > loss_text_ce: 0.03144017979502678 (0.04372134659021822) + | > loss_mel_ce: 3.1191675662994385 (3.4691466766527244) + | > loss: 0.7876519560813904 (0.878217005904378) + | > current_lr: 5e-06 + | > step_time: 0.1568 (0.24516117876288535) + | > loader_time: 0.0105 (0.011762456448598838) + + + --> TIME: 2024-10-20 02:32:15 -- STEP: 34850/69059 -- GLOBAL_STEP: 34850 + | > loss_text_ce: 0.03490259870886803 (0.043708622250465304) + | > loss_mel_ce: 3.1078600883483887 (3.4687754506804755) + | > loss: 0.7856906652450562 (0.8781210183293434) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.24516475273854763) + | > loader_time: 0.0107 (0.011762335016532469) + + + --> TIME: 2024-10-20 02:32:34 -- STEP: 34900/69059 -- GLOBAL_STEP: 34900 + | > loss_text_ce: 0.03595766797661781 (0.04369604022693098) + | > loss_mel_ce: 3.451277494430542 (3.4684362977281933) + | > loss: 0.8718087673187256 (0.8780330845824623) + | > current_lr: 5e-06 + | > step_time: 0.1812 (0.24516657293013652) + | > loader_time: 0.011 (0.011761951808601533) + + + --> TIME: 2024-10-20 02:32:52 -- STEP: 34950/69059 -- GLOBAL_STEP: 34950 + | > loss_text_ce: 0.03339291363954544 (0.043683082723299765) + | > loss_mel_ce: 3.5010666847229004 (3.4680662252497) + | > loss: 0.8836148977279663 (0.8779373270861578) + | > current_lr: 5e-06 + | > step_time: 0.2803 (0.24515014114298023) + | > loader_time: 0.012 (0.011760990418419137) + + + --> TIME: 2024-10-20 02:33:10 -- STEP: 35000/69059 -- GLOBAL_STEP: 35000 + | > loss_text_ce: 0.03679788485169411 (0.043670412289936476) + | > loss_mel_ce: 3.1839168071746826 (3.4676849013055984) + | > loss: 0.805178701877594 (0.8778388284870557) + | > current_lr: 5e-06 + | > step_time: 0.1644 (0.24514922475814818) + | > loader_time: 0.0107 (0.011760461425781257) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_35000.pth + + --> TIME: 2024-10-20 02:33:45 -- STEP: 35050/69059 -- GLOBAL_STEP: 35050 + | > loss_text_ce: 0.03706599026918411 (0.0436576490320487) + | > loss_mel_ce: 3.0200400352478027 (3.467369091657013) + | > loss: 0.7642765045166016 (0.8777566852549172) + | > current_lr: 5e-06 + | > step_time: 0.2349 (0.24513610935755362) + | > loader_time: 0.0109 (0.011760390944215614) + + + --> TIME: 2024-10-20 02:34:03 -- STEP: 35100/69059 -- GLOBAL_STEP: 35100 + | > loss_text_ce: 0.03839268162846565 (0.0436455309269497) + | > loss_mel_ce: 3.2639989852905273 (3.467013832823155) + | > loss: 0.8255979418754578 (0.8776648410161335) + | > current_lr: 5e-06 + | > step_time: 0.276 (0.24513239935931996) + | > loader_time: 0.0129 (0.011760214808320053) + + + --> TIME: 2024-10-20 02:34:21 -- STEP: 35150/69059 -- GLOBAL_STEP: 35150 + | > loss_text_ce: 0.03407662361860275 (0.043633300797115436) + | > loss_mel_ce: 3.2903738021850586 (3.46665250437696) + | > loss: 0.8311126232147217 (0.8775714513740025) + | > current_lr: 5e-06 + | > step_time: 0.1804 (0.24511048264049026) + | > loader_time: 0.0104 (0.01175985680196228) + + + --> TIME: 2024-10-20 02:34:39 -- STEP: 35200/69059 -- GLOBAL_STEP: 35200 + | > loss_text_ce: 0.03592386841773987 (0.04362092493599349) + | > loss_mel_ce: 3.312516450881958 (3.466327943361607) + | > loss: 0.8371101021766663 (0.8774872171574021) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.245099846734242) + | > loader_time: 0.0109 (0.011759960001165224) + + + --> TIME: 2024-10-20 02:34:58 -- STEP: 35250/69059 -- GLOBAL_STEP: 35250 + | > loss_text_ce: 0.03454264998435974 (0.04360836963730696) + | > loss_mel_ce: 3.137441635131836 (3.466003357359693) + | > loss: 0.7929960489273071 (0.877402931825489) + | > current_lr: 5e-06 + | > step_time: 0.1953 (0.24511784288392846) + | > loader_time: 0.0124 (0.011759963644311791) + + + --> TIME: 2024-10-20 02:35:16 -- STEP: 35300/69059 -- GLOBAL_STEP: 35300 + | > loss_text_ce: 0.034797776490449905 (0.04359596241460752) + | > loss_mel_ce: 3.2347805500030518 (3.4656894664723783) + | > loss: 0.8173945546150208 (0.8773213572947905) + | > current_lr: 5e-06 + | > step_time: 0.1575 (0.24511404222537034) + | > loader_time: 0.0103 (0.011759335001872559) + + + --> TIME: 2024-10-20 02:35:34 -- STEP: 35350/69059 -- GLOBAL_STEP: 35350 + | > loss_text_ce: 0.032458893954753876 (0.04358327321803029) + | > loss_mel_ce: 3.136502504348755 (3.4653932443265254) + | > loss: 0.7922403216362 (0.877244129460806) + | > current_lr: 5e-06 + | > step_time: 0.1861 (0.24509063966877895) + | > loader_time: 0.0104 (0.011759049916031343) + + + --> TIME: 2024-10-20 02:35:52 -- STEP: 35400/69059 -- GLOBAL_STEP: 35400 + | > loss_text_ce: 0.03330667316913605 (0.04357089419849196) + | > loss_mel_ce: 3.155426502227783 (3.4650624839127975) + | > loss: 0.7971832752227783 (0.8771583445984772) + | > current_lr: 5e-06 + | > step_time: 0.165 (0.24507896344540483) + | > loader_time: 0.0107 (0.01175822405491846) + + + --> TIME: 2024-10-20 02:36:10 -- STEP: 35450/69059 -- GLOBAL_STEP: 35450 + | > loss_text_ce: 0.03632719814777374 (0.04355847367073675) + | > loss_mel_ce: 3.337747812271118 (3.4646920476881067) + | > loss: 0.8435187339782715 (0.8770626304079019) + | > current_lr: 5e-06 + | > step_time: 0.2831 (0.24507794924952583) + | > loader_time: 0.0117 (0.011757621294352514) + + + --> TIME: 2024-10-20 02:36:29 -- STEP: 35500/69059 -- GLOBAL_STEP: 35500 + | > loss_text_ce: 0.03330682963132858 (0.04354623262049007) + | > loss_mel_ce: 3.1118204593658447 (3.464321973780501) + | > loss: 0.7862818241119385 (0.8769670516689072) + | > current_lr: 5e-06 + | > step_time: 0.2513 (0.24507533139913854) + | > loader_time: 0.0122 (0.011757114531288693) + + + --> TIME: 2024-10-20 02:36:49 -- STEP: 35550/69059 -- GLOBAL_STEP: 35550 + | > loss_text_ce: 0.033536337316036224 (0.04353400661613976) + | > loss_mel_ce: 2.9069619178771973 (3.4640411043167) + | > loss: 0.7351245880126953 (0.8768937778020217) + | > current_lr: 5e-06 + | > step_time: 0.1966 (0.2451046743272226) + | > loader_time: 0.0112 (0.011757266370556033) + + + --> TIME: 2024-10-20 02:37:06 -- STEP: 35600/69059 -- GLOBAL_STEP: 35600 + | > loss_text_ce: 0.0387360118329525 (0.043521870128716014) + | > loss_mel_ce: 3.284282684326172 (3.463682021438394) + | > loss: 0.8307546973228455 (0.8768009729616439) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.24508183385548968) + | > loader_time: 0.0121 (0.01175694758302711) + + + --> TIME: 2024-10-20 02:37:24 -- STEP: 35650/69059 -- GLOBAL_STEP: 35650 + | > loss_text_ce: 0.03652156889438629 (0.043509444227183315) + | > loss_mel_ce: 3.287884473800659 (3.4633934631855814) + | > loss: 0.8311015367507935 (0.8767257269217971) + | > current_lr: 5e-06 + | > step_time: 0.2016 (0.24505905629541896) + | > loader_time: 0.0109 (0.01175599585743139) + + + --> TIME: 2024-10-20 02:37:42 -- STEP: 35700/69059 -- GLOBAL_STEP: 35700 + | > loss_text_ce: 0.03295045346021652 (0.04349753563347088) + | > loss_mel_ce: 3.016968011856079 (3.463074113274144) + | > loss: 0.7624796032905579 (0.8766429122973568) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.24505313220812158) + | > loader_time: 0.0103 (0.011755539409252787) + + + --> TIME: 2024-10-20 02:38:01 -- STEP: 35750/69059 -- GLOBAL_STEP: 35750 + | > loss_text_ce: 0.03413375839591026 (0.04348556770191516) + | > loss_mel_ce: 3.058461904525757 (3.462744539014099) + | > loss: 0.7731488943099976 (0.8765575267451637) + | > current_lr: 5e-06 + | > step_time: 0.2159 (0.24507178551333766) + | > loader_time: 0.0111 (0.011755203667220547) + + + --> TIME: 2024-10-20 02:38:20 -- STEP: 35800/69059 -- GLOBAL_STEP: 35800 + | > loss_text_ce: 0.03534423187375069 (0.04347348380474843) + | > loss_mel_ce: 3.420090913772583 (3.4623990877114097) + | > loss: 0.8638587594032288 (0.8764681429436763) + | > current_lr: 5e-06 + | > step_time: 0.2513 (0.24507749185881803) + | > loader_time: 0.0114 (0.011754554476817907) + + + --> TIME: 2024-10-20 02:38:39 -- STEP: 35850/69059 -- GLOBAL_STEP: 35850 + | > loss_text_ce: 0.03548106923699379 (0.04346159233936352) + | > loss_mel_ce: 3.5824246406555176 (3.4620684668608694) + | > loss: 0.9044764041900635 (0.8763825148651635) + | > current_lr: 5e-06 + | > step_time: 0.2318 (0.24508006871328528) + | > loader_time: 0.0125 (0.011754503469586883) + + + --> TIME: 2024-10-20 02:38:56 -- STEP: 35900/69059 -- GLOBAL_STEP: 35900 + | > loss_text_ce: 0.0361131951212883 (0.04344894666987396) + | > loss_mel_ce: 3.591973066329956 (3.4617472705070607) + | > loss: 0.9070215821266174 (0.8762990543636449) + | > current_lr: 5e-06 + | > step_time: 0.165 (0.24505643670605418) + | > loader_time: 0.0112 (0.011753948007121399) + + + --> TIME: 2024-10-20 02:39:15 -- STEP: 35950/69059 -- GLOBAL_STEP: 35950 + | > loss_text_ce: 0.03179476037621498 (0.04343654988439777) + | > loss_mel_ce: 3.2556023597717285 (3.461404309896162) + | > loss: 0.8218492865562439 (0.876210215013781) + | > current_lr: 5e-06 + | > step_time: 0.1778 (0.24505509412338405) + | > loader_time: 0.01 (0.011753733526184768) + + + --> TIME: 2024-10-20 02:39:32 -- STEP: 36000/69059 -- GLOBAL_STEP: 36000 + | > loss_text_ce: 0.03548211604356766 (0.0434244532732084) + | > loss_mel_ce: 3.3235485553741455 (3.4610892336633383) + | > loss: 0.8397576808929443 (0.876128421796694) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24502861310376062) + | > loader_time: 0.0104 (0.01175284595621957) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_36000.pth + + --> TIME: 2024-10-20 02:40:05 -- STEP: 36050/69059 -- GLOBAL_STEP: 36050 + | > loss_text_ce: 0.03283124789595604 (0.04341182764784907) + | > loss_mel_ce: 3.031024694442749 (3.46080384307363) + | > loss: 0.7659639716148376 (0.8760539177419754) + | > current_lr: 5e-06 + | > step_time: 0.1773 (0.2450326806728453) + | > loader_time: 0.012 (0.011752817019014187) + + + --> TIME: 2024-10-20 02:40:23 -- STEP: 36100/69059 -- GLOBAL_STEP: 36100 + | > loss_text_ce: 0.032686226069927216 (0.04339947824245119) + | > loss_mel_ce: 3.1772408485412598 (3.460424929999239) + | > loss: 0.8024817705154419 (0.8759561021209108) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.24502459671358653) + | > loader_time: 0.012 (0.01175277511168715) + + + --> TIME: 2024-10-20 02:40:42 -- STEP: 36150/69059 -- GLOBAL_STEP: 36150 + | > loss_text_ce: 0.0325099341571331 (0.04338669363672041) + | > loss_mel_ce: 3.352280378341675 (3.4601161705837438) + | > loss: 0.8461976051330566 (0.8758757161154307) + | > current_lr: 5e-06 + | > step_time: 0.2315 (0.24502501957973832) + | > loader_time: 0.0111 (0.011752985865908214) + + + --> TIME: 2024-10-20 02:41:00 -- STEP: 36200/69059 -- GLOBAL_STEP: 36200 + | > loss_text_ce: 0.03296737000346184 (0.043374794195122954) + | > loss_mel_ce: 3.310741901397705 (3.459798936652863) + | > loss: 0.8359273076057434 (0.8757934327761128) + | > current_lr: 5e-06 + | > step_time: 0.1742 (0.24502167496233354) + | > loader_time: 0.011 (0.011752637539120673) + + + --> TIME: 2024-10-20 02:41:19 -- STEP: 36250/69059 -- GLOBAL_STEP: 36250 + | > loss_text_ce: 0.04061156511306763 (0.04336292221946945) + | > loss_mel_ce: 3.126542806625366 (3.4594363211730315) + | > loss: 0.7917885780334473 (0.8756998109143369) + | > current_lr: 5e-06 + | > step_time: 0.1955 (0.2450243471671795) + | > loader_time: 0.011 (0.01175271559748157) + + + --> TIME: 2024-10-20 02:41:38 -- STEP: 36300/69059 -- GLOBAL_STEP: 36300 + | > loss_text_ce: 0.03433699160814285 (0.04335072668387277) + | > loss_mel_ce: 2.9362781047821045 (3.459104461091925) + | > loss: 0.7426537871360779 (0.8756137970114553) + | > current_lr: 5e-06 + | > step_time: 0.1566 (0.2450336851561365) + | > loader_time: 0.0124 (0.011752795910375513) + + + --> TIME: 2024-10-20 02:41:58 -- STEP: 36350/69059 -- GLOBAL_STEP: 36350 + | > loss_text_ce: 0.036675650626420975 (0.04333924655957535) + | > loss_mel_ce: 3.549252510070801 (3.4588155180486275) + | > loss: 0.8964820504188538 (0.875538691216683) + | > current_lr: 5e-06 + | > step_time: 0.1778 (0.24507574752403585) + | > loader_time: 0.0108 (0.011753168447145575) + + + --> TIME: 2024-10-20 02:42:17 -- STEP: 36400/69059 -- GLOBAL_STEP: 36400 + | > loss_text_ce: 0.033393289893865585 (0.04332793173861187) + | > loss_mel_ce: 3.123725175857544 (3.458495740504046) + | > loss: 0.7892796397209167 (0.8754559181286743) + | > current_lr: 5e-06 + | > step_time: 0.164 (0.24507669103669596) + | > loader_time: 0.0102 (0.011752832656378278) + + + --> TIME: 2024-10-20 02:42:35 -- STEP: 36450/69059 -- GLOBAL_STEP: 36450 + | > loss_text_ce: 0.03507950156927109 (0.04331616129322448) + | > loss_mel_ce: 3.2348415851593018 (3.4581283387758335) + | > loss: 0.8174802660942078 (0.8753611250832908) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.2450703268482852) + | > loader_time: 0.0106 (0.011752330259367593) + + + --> TIME: 2024-10-20 02:42:54 -- STEP: 36500/69059 -- GLOBAL_STEP: 36500 + | > loss_text_ce: 0.03382536768913269 (0.0433043512498881) + | > loss_mel_ce: 3.4754600524902344 (3.457771518635413) + | > loss: 0.8773213624954224 (0.8752689675353982) + | > current_lr: 5e-06 + | > step_time: 0.2044 (0.24507392781401333) + | > loader_time: 0.0117 (0.011752071413275306) + + + --> TIME: 2024-10-20 02:43:14 -- STEP: 36550/69059 -- GLOBAL_STEP: 36550 + | > loss_text_ce: 0.03567138686776161 (0.04329213938419714) + | > loss_mel_ce: 3.043388605117798 (3.4574763368027206) + | > loss: 0.7697650194168091 (0.8751921191114408) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.24509964649393523) + | > loader_time: 0.0111 (0.011752706519882253) + + + --> TIME: 2024-10-20 02:43:34 -- STEP: 36600/69059 -- GLOBAL_STEP: 36600 + | > loss_text_ce: 0.0348697192966938 (0.043280859776451706) + | > loss_mel_ce: 3.059474229812622 (3.45721557707082) + | > loss: 0.7735859751701355 (0.8751241092701431) + | > current_lr: 5e-06 + | > step_time: 0.181 (0.24512932320110134) + | > loader_time: 0.0111 (0.011753041907086387) + + + --> TIME: 2024-10-20 02:43:52 -- STEP: 36650/69059 -- GLOBAL_STEP: 36650 + | > loss_text_ce: 0.03641579672694206 (0.04326947570990591) + | > loss_mel_ce: 3.309023141860962 (3.4569145410935747) + | > loss: 0.8363597393035889 (0.8750460042602209) + | > current_lr: 5e-06 + | > step_time: 0.3756 (0.24513317438622614) + | > loader_time: 0.0133 (0.011752681959862583) + + + --> TIME: 2024-10-20 02:44:10 -- STEP: 36700/69059 -- GLOBAL_STEP: 36700 + | > loss_text_ce: 0.034618787467479706 (0.043257932539379394) + | > loss_mel_ce: 3.031611680984497 (3.456543021975152) + | > loss: 0.7665576338768005 (0.8749502386855177) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.24511445030529427) + | > loader_time: 0.011 (0.011752215269800739) + + + --> TIME: 2024-10-20 02:44:29 -- STEP: 36750/69059 -- GLOBAL_STEP: 36750 + | > loss_text_ce: 0.03488939255475998 (0.043246107187248556) + | > loss_mel_ce: 3.1714694499969482 (3.4562804923089914) + | > loss: 0.8015897274017334 (0.8748816499272178) + | > current_lr: 5e-06 + | > step_time: 0.1798 (0.2451166646594093) + | > loader_time: 0.0116 (0.011751967430114736) + + + --> TIME: 2024-10-20 02:44:48 -- STEP: 36800/69059 -- GLOBAL_STEP: 36800 + | > loss_text_ce: 0.0344693697988987 (0.04323433164557741) + | > loss_mel_ce: 3.3254880905151367 (3.4559596715543495) + | > loss: 0.8399893641471863 (0.8747985008520934) + | > current_lr: 5e-06 + | > step_time: 0.3335 (0.24513342775080516) + | > loader_time: 0.0121 (0.011751737957415366) + + + --> TIME: 2024-10-20 02:45:07 -- STEP: 36850/69059 -- GLOBAL_STEP: 36850 + | > loss_text_ce: 0.03372177109122276 (0.043221717507324196) + | > loss_mel_ce: 3.4354472160339355 (3.455661337521206) + | > loss: 0.8672922253608704 (0.8747207638091611) + | > current_lr: 5e-06 + | > step_time: 0.2632 (0.24514164902623667) + | > loader_time: 0.0108 (0.011751691761301513) + + + --> TIME: 2024-10-20 02:45:26 -- STEP: 36900/69059 -- GLOBAL_STEP: 36900 + | > loss_text_ce: 0.034449707716703415 (0.04320983942229587) + | > loss_mel_ce: 3.2048513889312744 (3.455361574568389) + | > loss: 0.8098253011703491 (0.8746428535511188) + | > current_lr: 5e-06 + | > step_time: 0.3321 (0.24514560083063638) + | > loader_time: 0.0128 (0.011751297056513423) + + + --> TIME: 2024-10-20 02:45:43 -- STEP: 36950/69059 -- GLOBAL_STEP: 36950 + | > loss_text_ce: 0.035998087376356125 (0.043198464336238945) + | > loss_mel_ce: 3.2430505752563477 (3.4550202237607905) + | > loss: 0.8197621703147888 (0.8745546720782876) + | > current_lr: 5e-06 + | > step_time: 0.2017 (0.24512395045754068) + | > loader_time: 0.0103 (0.011750716324265496) + + + --> TIME: 2024-10-20 02:46:02 -- STEP: 37000/69059 -- GLOBAL_STEP: 37000 + | > loss_text_ce: 0.033515095710754395 (0.04318697729424826) + | > loss_mel_ce: 3.497670888900757 (3.4546991288855176) + | > loss: 0.8827965259552002 (0.8744715265986095) + | > current_lr: 5e-06 + | > step_time: 0.1573 (0.2451260147739101) + | > loader_time: 0.0104 (0.01175030332642632) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_37000.pth + + --> TIME: 2024-10-20 02:46:39 -- STEP: 37050/69059 -- GLOBAL_STEP: 37050 + | > loss_text_ce: 0.033966582268476486 (0.043175844345668866) + | > loss_mel_ce: 3.0752220153808594 (3.454441928754121) + | > loss: 0.7772971391677856 (0.8744044433367248) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.24514169118986762) + | > loader_time: 0.0105 (0.0117504281392786) + + + --> TIME: 2024-10-20 02:46:58 -- STEP: 37100/69059 -- GLOBAL_STEP: 37100 + | > loss_text_ce: 0.03261352330446243 (0.04316391380815025) + | > loss_mel_ce: 3.064387559890747 (3.4541522188199383) + | > loss: 0.7742502689361572 (0.8743290332208761) + | > current_lr: 5e-06 + | > step_time: 0.1509 (0.245141038207031) + | > loader_time: 0.0098 (0.011750498004358085) + + + --> TIME: 2024-10-20 02:47:16 -- STEP: 37150/69059 -- GLOBAL_STEP: 37150 + | > loss_text_ce: 0.03503187745809555 (0.04315255892203537) + | > loss_mel_ce: 3.0878052711486816 (3.4537856473126807) + | > loss: 0.7807092666625977 (0.8742345516197149) + | > current_lr: 5e-06 + | > step_time: 0.2311 (0.2451217899219826) + | > loader_time: 0.0114 (0.011749915377127848) + + + --> TIME: 2024-10-20 02:47:34 -- STEP: 37200/69059 -- GLOBAL_STEP: 37200 + | > loss_text_ce: 0.03915732726454735 (0.04314049390394978) + | > loss_mel_ce: 3.244594097137451 (3.453451525126721) + | > loss: 0.8209378719329834 (0.8741480048241166) + | > current_lr: 5e-06 + | > step_time: 0.324 (0.24511186486290348) + | > loader_time: 0.0132 (0.01174988189692138) + + + --> TIME: 2024-10-20 02:47:52 -- STEP: 37250/69059 -- GLOBAL_STEP: 37250 + | > loss_text_ce: 0.03594416007399559 (0.04312894723799381) + | > loss_mel_ce: 3.0587782859802246 (3.4530277039316557) + | > loss: 0.7736806273460388 (0.874039162859822) + | > current_lr: 5e-06 + | > step_time: 0.1779 (0.24510305245930716) + | > loader_time: 0.0107 (0.011749907250372353) + + + --> TIME: 2024-10-20 02:48:11 -- STEP: 37300/69059 -- GLOBAL_STEP: 37300 + | > loss_text_ce: 0.03489600494503975 (0.04311770303359596) + | > loss_mel_ce: 3.2710251808166504 (3.4527060019873934) + | > loss: 0.8264802694320679 (0.8739559263214681) + | > current_lr: 5e-06 + | > step_time: 0.1651 (0.2451043865188517) + | > loader_time: 0.011 (0.011749754150175856) + + + --> TIME: 2024-10-20 02:48:28 -- STEP: 37350/69059 -- GLOBAL_STEP: 37350 + | > loss_text_ce: 0.03468199074268341 (0.043106617342176766) + | > loss_mel_ce: 3.3831660747528076 (3.4523882401133035) + | > loss: 0.8544620275497437 (0.8738737144288303) + | > current_lr: 5e-06 + | > step_time: 0.1966 (0.24508271992286365) + | > loader_time: 0.0113 (0.011749307212421053) + + + --> TIME: 2024-10-20 02:48:48 -- STEP: 37400/69059 -- GLOBAL_STEP: 37400 + | > loss_text_ce: 0.0343327596783638 (0.043095758226525595) + | > loss_mel_ce: 3.63865327835083 (3.452112074548536) + | > loss: 0.9182465076446533 (0.8738019582597343) + | > current_lr: 5e-06 + | > step_time: 0.5032 (0.2450989237571145) + | > loader_time: 0.0146 (0.01174927511317207) + + + --> TIME: 2024-10-20 02:49:06 -- STEP: 37450/69059 -- GLOBAL_STEP: 37450 + | > loss_text_ce: 0.033885348588228226 (0.04308497512521566) + | > loss_mel_ce: 3.210452079772949 (3.4517651708421986) + | > loss: 0.8110843300819397 (0.8737125365644985) + | > current_lr: 5e-06 + | > step_time: 0.2014 (0.24509662783511968) + | > loader_time: 0.0108 (0.011749300020877443) + + + --> TIME: 2024-10-20 02:49:22 -- STEP: 37500/69059 -- GLOBAL_STEP: 37500 + | > loss_text_ce: 0.037589650601148605 (0.04307301629126049) + | > loss_mel_ce: 3.396898031234741 (3.451415727532692) + | > loss: 0.8586218953132629 (0.8736221860281637) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.2450554986254374) + | > loader_time: 0.011 (0.011748605168660477) + + + --> TIME: 2024-10-20 02:49:42 -- STEP: 37550/69059 -- GLOBAL_STEP: 37550 + | > loss_text_ce: 0.03748089075088501 (0.04306205630843097) + | > loss_mel_ce: 3.175905704498291 (3.4511690643147865) + | > loss: 0.8033466339111328 (0.8735577802239035) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.24506622701764266) + | > loader_time: 0.0108 (0.011748273610751253) + + + --> TIME: 2024-10-20 02:50:00 -- STEP: 37600/69059 -- GLOBAL_STEP: 37600 + | > loss_text_ce: 0.0364924892783165 (0.043051030978391816) + | > loss_mel_ce: 3.000135660171509 (3.450889202362663) + | > loss: 0.7591570615768433 (0.8734850584018116) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.2450690854039598) + | > loader_time: 0.0111 (0.011748097120447356) + + + --> TIME: 2024-10-20 02:50:19 -- STEP: 37650/69059 -- GLOBAL_STEP: 37650 + | > loss_text_ce: 0.034902218729257584 (0.04304032255201561) + | > loss_mel_ce: 3.1348800659179688 (3.4506046030467377) + | > loss: 0.7924456000328064 (0.8734112314636501) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24507868254485515) + | > loader_time: 0.0114 (0.011748089537044284) + + + --> TIME: 2024-10-20 02:50:38 -- STEP: 37700/69059 -- GLOBAL_STEP: 37700 + | > loss_text_ce: 0.03457982465624809 (0.04302868717989866) + | > loss_mel_ce: 3.516770601272583 (3.450300294480191) + | > loss: 0.8878375887870789 (0.8733322454763672) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.24507947764915244) + | > loader_time: 0.0116 (0.01174780387144822) + + + --> TIME: 2024-10-20 02:50:56 -- STEP: 37750/69059 -- GLOBAL_STEP: 37750 + | > loss_text_ce: 0.03782276064157486 (0.04301721073657453) + | > loss_mel_ce: 3.325025796890259 (3.450003430404391) + | > loss: 0.8407121300697327 (0.8732551603506744) + | > current_lr: 5e-06 + | > step_time: 0.1855 (0.24508032455823278) + | > loader_time: 0.011 (0.011747390696544511) + + + --> TIME: 2024-10-20 02:51:17 -- STEP: 37800/69059 -- GLOBAL_STEP: 37800 + | > loss_text_ce: 0.036752961575984955 (0.04300573479517209) + | > loss_mel_ce: 3.6635613441467285 (3.4497046033793457) + | > loss: 0.9250785708427429 (0.8731775846115507) + | > current_lr: 5e-06 + | > step_time: 0.3459 (0.24511485996069732) + | > loader_time: 0.0126 (0.011747783895522819) + + + --> TIME: 2024-10-20 02:51:37 -- STEP: 37850/69059 -- GLOBAL_STEP: 37850 + | > loss_text_ce: 0.030054189264774323 (0.042994490775428336) + | > loss_mel_ce: 3.2784156799316406 (3.449483475168577) + | > loss: 0.8271174430847168 (0.873119491548174) + | > current_lr: 5e-06 + | > step_time: 0.1941 (0.24514095036797667) + | > loader_time: 0.0106 (0.011747652177294138) + + + --> TIME: 2024-10-20 02:51:57 -- STEP: 37900/69059 -- GLOBAL_STEP: 37900 + | > loss_text_ce: 0.03392401710152626 (0.04298378055423749) + | > loss_mel_ce: 3.382874011993408 (3.449191845664855) + | > loss: 0.8541995286941528 (0.8730439066147753) + | > current_lr: 5e-06 + | > step_time: 0.3899 (0.24517709820754924) + | > loader_time: 0.0134 (0.01174766643066204) + + + --> TIME: 2024-10-20 02:52:16 -- STEP: 37950/69059 -- GLOBAL_STEP: 37950 + | > loss_text_ce: 0.031717877835035324 (0.04297163832255361) + | > loss_mel_ce: 3.059144973754883 (3.448919762459333) + | > loss: 0.77271568775177 (0.8729728502482776) + | > current_lr: 5e-06 + | > step_time: 0.167 (0.24518183855828246) + | > loader_time: 0.0102 (0.01174730734391645) + + + --> TIME: 2024-10-20 02:52:35 -- STEP: 38000/69059 -- GLOBAL_STEP: 38000 + | > loss_text_ce: 0.03049013577401638 (0.042960492863662686) + | > loss_mel_ce: 3.4101428985595703 (3.448665776654271) + | > loss: 0.8601582646369934 (0.8729065674308111) + | > current_lr: 5e-06 + | > step_time: 0.1642 (0.24518469378195312) + | > loader_time: 0.0102 (0.011746716932246546) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_38000.pth + + --> TIME: 2024-10-20 02:53:10 -- STEP: 38050/69059 -- GLOBAL_STEP: 38050 + | > loss_text_ce: 0.03637011721730232 (0.04294982540260844) + | > loss_mel_ce: 3.1951398849487305 (3.4483378017653386) + | > loss: 0.8078774809837341 (0.8728219068410686) + | > current_lr: 5e-06 + | > step_time: 0.3288 (0.24516375329272036) + | > loader_time: 0.0112 (0.011746385163295897) + + + --> TIME: 2024-10-20 02:53:29 -- STEP: 38100/69059 -- GLOBAL_STEP: 38100 + | > loss_text_ce: 0.034996397793293 (0.042938374187357076) + | > loss_mel_ce: 3.2542154788970947 (3.448006130873039) + | > loss: 0.8223029971122742 (0.8727361263190053) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.2451638744321708) + | > loader_time: 0.0111 (0.011746699259349977) + + + --> TIME: 2024-10-20 02:53:46 -- STEP: 38150/69059 -- GLOBAL_STEP: 38150 + | > loss_text_ce: 0.0336124561727047 (0.04292693317766482) + | > loss_mel_ce: 3.205655336380005 (3.447688056200837) + | > loss: 0.8098169565200806 (0.8726537473954828) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.2451432757046401) + | > loader_time: 0.0105 (0.011746614219945484) + + + --> TIME: 2024-10-20 02:54:04 -- STEP: 38200/69059 -- GLOBAL_STEP: 38200 + | > loss_text_ce: 0.034220658242702484 (0.042915764828139205) + | > loss_mel_ce: 2.931030511856079 (3.4473740973085403) + | > loss: 0.7413128018379211 (0.8725724655889103) + | > current_lr: 5e-06 + | > step_time: 0.152 (0.24512843914056948) + | > loader_time: 0.0107 (0.011746318877055363) + + + --> TIME: 2024-10-20 02:54:23 -- STEP: 38250/69059 -- GLOBAL_STEP: 38250 + | > loss_text_ce: 0.037337109446525574 (0.04290533886849857) + | > loss_mel_ce: 3.6985132694244385 (3.447139506570631) + | > loss: 0.9339625835418701 (0.8725112114139643) + | > current_lr: 5e-06 + | > step_time: 0.2851 (0.2451375421636245) + | > loader_time: 0.0126 (0.011746572251413371) + + + --> TIME: 2024-10-20 02:54:42 -- STEP: 38300/69059 -- GLOBAL_STEP: 38300 + | > loss_text_ce: 0.032345566898584366 (0.042894678674346244) + | > loss_mel_ce: 3.1810598373413086 (3.4468417355472445) + | > loss: 0.8033513426780701 (0.8724341036071988) + | > current_lr: 5e-06 + | > step_time: 0.1816 (0.2451471587572023) + | > loader_time: 0.0106 (0.011746724847091385) + + + --> TIME: 2024-10-20 02:55:01 -- STEP: 38350/69059 -- GLOBAL_STEP: 38350 + | > loss_text_ce: 0.03185317665338516 (0.042883621468323835) + | > loss_mel_ce: 2.69701886177063 (3.446499698569264) + | > loss: 0.6822180151939392 (0.8723458300637775) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.24515832160566745) + | > loader_time: 0.0112 (0.011746538757811613) + + + --> TIME: 2024-10-20 02:55:19 -- STEP: 38400/69059 -- GLOBAL_STEP: 38400 + | > loss_text_ce: 0.03408791869878769 (0.04287197002175743) + | > loss_mel_ce: 2.974860429763794 (3.4461084743030144) + | > loss: 0.75223708152771 (0.8722451111394923) + | > current_lr: 5e-06 + | > step_time: 0.181 (0.2451413652362923) + | > loader_time: 0.0108 (0.011746358592063178) + + + --> TIME: 2024-10-20 02:55:36 -- STEP: 38450/69059 -- GLOBAL_STEP: 38450 + | > loss_text_ce: 0.035300638526678085 (0.04286108699829448) + | > loss_mel_ce: 3.2208786010742188 (3.445785674044285) + | > loss: 0.8140448331832886 (0.8721616903207093) + | > current_lr: 5e-06 + | > step_time: 0.2467 (0.24512456333373706) + | > loader_time: 0.0109 (0.011745979212350422) + + + --> TIME: 2024-10-20 02:55:56 -- STEP: 38500/69059 -- GLOBAL_STEP: 38500 + | > loss_text_ce: 0.03657351806759834 (0.042850386893981254) + | > loss_mel_ce: 3.204253673553467 (3.4454556991837015) + | > loss: 0.8102067708969116 (0.8720765215808707) + | > current_lr: 5e-06 + | > step_time: 0.4429 (0.24514899183248545) + | > loader_time: 0.0136 (0.011745800303174288) + + + --> TIME: 2024-10-20 02:56:15 -- STEP: 38550/69059 -- GLOBAL_STEP: 38550 + | > loss_text_ce: 0.03437596559524536 (0.04283969209574132) + | > loss_mel_ce: 3.211294412612915 (3.445147976312479) + | > loss: 0.8114175796508789 (0.8719969171587143) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.24515886561570618) + | > loader_time: 0.0107 (0.011745799444374242) + + + --> TIME: 2024-10-20 02:56:33 -- STEP: 38600/69059 -- GLOBAL_STEP: 38600 + | > loss_text_ce: 0.034281175583601 (0.042828465321781424) + | > loss_mel_ce: 3.12793231010437 (3.444823478707355) + | > loss: 0.7905533909797668 (0.8719129860679113) + | > current_lr: 5e-06 + | > step_time: 0.1568 (0.24514168717082918) + | > loader_time: 0.0108 (0.01174530893409807) + + + --> TIME: 2024-10-20 02:56:52 -- STEP: 38650/69059 -- GLOBAL_STEP: 38650 + | > loss_text_ce: 0.03412698209285736 (0.042817399965654396) + | > loss_mel_ce: 3.3380045890808105 (3.4444536601247724) + | > loss: 0.8430328965187073 (0.8718177650847699) + | > current_lr: 5e-06 + | > step_time: 0.4042 (0.2451482978613534) + | > loader_time: 0.0134 (0.011745207201771402) + + + --> TIME: 2024-10-20 02:57:09 -- STEP: 38700/69059 -- GLOBAL_STEP: 38700 + | > loss_text_ce: 0.031726911664009094 (0.04280701874141641) + | > loss_mel_ce: 3.116006851196289 (3.4440903365889035) + | > loss: 0.786933422088623 (0.8717243388949758) + | > current_lr: 5e-06 + | > step_time: 0.1968 (0.2451182560711252) + | > loader_time: 0.0106 (0.011744279510291038) + + + --> TIME: 2024-10-20 02:57:27 -- STEP: 38750/69059 -- GLOBAL_STEP: 38750 + | > loss_text_ce: 0.0350012481212616 (0.04279621418978876) + | > loss_mel_ce: 3.1032159328460693 (3.44383150573236) + | > loss: 0.7845543026924133 (0.8716569300420834) + | > current_lr: 5e-06 + | > step_time: 0.2119 (0.24511165239887853) + | > loader_time: 0.01 (0.011743522121060265) + + + --> TIME: 2024-10-20 02:57:47 -- STEP: 38800/69059 -- GLOBAL_STEP: 38800 + | > loss_text_ce: 0.03330395743250847 (0.04278549521393361) + | > loss_mel_ce: 3.2641539573669434 (3.4435683306713467) + | > loss: 0.8243644833564758 (0.8715884565323909) + | > current_lr: 5e-06 + | > step_time: 0.1586 (0.24513664770372134) + | > loader_time: 0.0115 (0.011743206861092854) + + + --> TIME: 2024-10-20 02:58:06 -- STEP: 38850/69059 -- GLOBAL_STEP: 38850 + | > loss_text_ce: 0.0337994359433651 (0.04277483823293251) + | > loss_mel_ce: 3.0977721214294434 (3.4432886528140103) + | > loss: 0.7828928828239441 (0.8715158728221527) + | > current_lr: 5e-06 + | > step_time: 0.3029 (0.2451545408977965) + | > loader_time: 0.0104 (0.011743027896954725) + + + --> TIME: 2024-10-20 02:58:25 -- STEP: 38900/69059 -- GLOBAL_STEP: 38900 + | > loss_text_ce: 0.03242266923189163 (0.04276405848428626) + | > loss_mel_ce: 2.958449125289917 (3.442939237880194) + | > loss: 0.7477179765701294 (0.8714258241484906) + | > current_lr: 5e-06 + | > step_time: 0.1498 (0.24516202821829633) + | > loader_time: 0.0129 (0.011742725666507017) + + + --> TIME: 2024-10-20 02:58:44 -- STEP: 38950/69059 -- GLOBAL_STEP: 38950 + | > loss_text_ce: 0.03783826529979706 (0.04275341456375797) + | > loss_mel_ce: 3.490568161010742 (3.4426224277628332) + | > loss: 0.8821015954017639 (0.8713439606441931) + | > current_lr: 5e-06 + | > step_time: 0.2969 (0.24517116506231307) + | > loader_time: 0.0103 (0.011742274231720922) + + + --> TIME: 2024-10-20 02:59:04 -- STEP: 39000/69059 -- GLOBAL_STEP: 39000 + | > loss_text_ce: 0.0352969728410244 (0.04274300842579338) + | > loss_mel_ce: 3.107626438140869 (3.4424029025725607) + | > loss: 0.7857308387756348 (0.8712864778102983) + | > current_lr: 5e-06 + | > step_time: 0.1817 (0.24519924540397448) + | > loader_time: 0.0115 (0.011742187646719094) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_39000.pth + + --> TIME: 2024-10-20 02:59:39 -- STEP: 39050/69059 -- GLOBAL_STEP: 39050 + | > loss_text_ce: 0.034786928445100784 (0.042732391076713186) + | > loss_mel_ce: 3.157503366470337 (3.4420921602810624) + | > loss: 0.7980725765228271 (0.8712061378989432) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24521577605517358) + | > loader_time: 0.0116 (0.011742250940711156) + + + --> TIME: 2024-10-20 02:59:58 -- STEP: 39100/69059 -- GLOBAL_STEP: 39100 + | > loss_text_ce: 0.03538503870368004 (0.04272174353761309) + | > loss_mel_ce: 3.2789034843444824 (3.441764386157528) + | > loss: 0.828572154045105 (0.8711215324795166) + | > current_lr: 5e-06 + | > step_time: 0.2254 (0.2452126212498111) + | > loader_time: 0.0115 (0.01174223205927386) + + + --> TIME: 2024-10-20 03:00:17 -- STEP: 39150/69059 -- GLOBAL_STEP: 39150 + | > loss_text_ce: 0.03494945541024208 (0.042710962917745926) + | > loss_mel_ce: 3.2823853492736816 (3.4415220622114835) + | > loss: 0.8293337225914001 (0.8710582563383837) + | > current_lr: 5e-06 + | > step_time: 0.276 (0.24523689444798924) + | > loader_time: 0.0123 (0.011742634152087199) + + + --> TIME: 2024-10-20 03:00:37 -- STEP: 39200/69059 -- GLOBAL_STEP: 39200 + | > loss_text_ce: 0.03315109759569168 (0.04270096702282585) + | > loss_mel_ce: 2.9003636837005615 (3.441243477068363) + | > loss: 0.7333787083625793 (0.8709861110789446) + | > current_lr: 5e-06 + | > step_time: 0.2964 (0.24525574591695046) + | > loader_time: 0.0125 (0.01174280689078933) + + + --> TIME: 2024-10-20 03:00:55 -- STEP: 39250/69059 -- GLOBAL_STEP: 39250 + | > loss_text_ce: 0.03460870683193207 (0.04269056246818803) + | > loss_mel_ce: 2.9037623405456543 (3.440950681194356) + | > loss: 0.7345927357673645 (0.8709103109715103) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.2452540025650316) + | > loader_time: 0.0104 (0.01174288977483274) + + + --> TIME: 2024-10-20 03:01:14 -- STEP: 39300/69059 -- GLOBAL_STEP: 39300 + | > loss_text_ce: 0.03570752218365669 (0.042680410817976265) + | > loss_mel_ce: 3.247091054916382 (3.440628219384858) + | > loss: 0.8206996321678162 (0.8708271576020565) + | > current_lr: 5e-06 + | > step_time: 0.1729 (0.24525699320640273) + | > loader_time: 0.0107 (0.0117428584438547) + + + --> TIME: 2024-10-20 03:01:31 -- STEP: 39350/69059 -- GLOBAL_STEP: 39350 + | > loss_text_ce: 0.03165202587842941 (0.042669610489666124) + | > loss_mel_ce: 3.3723556995391846 (3.4402822578907397) + | > loss: 0.8510019183158875 (0.870737967142763) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.24521969105721733) + | > loader_time: 0.0112 (0.01174203642441627) + + + --> TIME: 2024-10-20 03:01:49 -- STEP: 39400/69059 -- GLOBAL_STEP: 39400 + | > loss_text_ce: 0.033918581902980804 (0.04265874458268873) + | > loss_mel_ce: 2.8514785766601562 (3.439956905115657) + | > loss: 0.72134929895401 (0.8706539124749649) + | > current_lr: 5e-06 + | > step_time: 0.153 (0.24522255565914405) + | > loader_time: 0.0116 (0.01174162939115222) + + + --> TIME: 2024-10-20 03:02:06 -- STEP: 39450/69059 -- GLOBAL_STEP: 39450 + | > loss_text_ce: 0.03588402643799782 (0.04264810807577994) + | > loss_mel_ce: 3.462250232696533 (3.439611861575821) + | > loss: 0.8745335936546326 (0.8705649924580449) + | > current_lr: 5e-06 + | > step_time: 0.3208 (0.245193613406521) + | > loader_time: 0.0114 (0.011741402478574624) + + + --> TIME: 2024-10-20 03:02:26 -- STEP: 39500/69059 -- GLOBAL_STEP: 39500 + | > loss_text_ce: 0.03855087608098984 (0.04263801525877417) + | > loss_mel_ce: 3.517584800720215 (3.4393586925735735) + | > loss: 0.8890339136123657 (0.8704991770062277) + | > current_lr: 5e-06 + | > step_time: 0.3622 (0.2452220203484161) + | > loader_time: 0.0157 (0.011741897667510576) + + + --> TIME: 2024-10-20 03:02:45 -- STEP: 39550/69059 -- GLOBAL_STEP: 39550 + | > loss_text_ce: 0.034402310848236084 (0.04262792040905914) + | > loss_mel_ce: 3.2606141567230225 (3.4391223633485035) + | > loss: 0.8237541317939758 (0.8704375709911676) + | > current_lr: 5e-06 + | > step_time: 0.2155 (0.24522433724626427) + | > loader_time: 0.0117 (0.011741489043880783) + + + --> TIME: 2024-10-20 03:03:04 -- STEP: 39600/69059 -- GLOBAL_STEP: 39600 + | > loss_text_ce: 0.032253433018922806 (0.042617760394097634) + | > loss_mel_ce: 3.185488224029541 (3.438874629892462) + | > loss: 0.8044354319572449 (0.8703730976235395) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24522116626151885) + | > loader_time: 0.0114 (0.011741151918064447) + + + --> TIME: 2024-10-20 03:03:23 -- STEP: 39650/69059 -- GLOBAL_STEP: 39650 + | > loss_text_ce: 0.0354376845061779 (0.042607976882277916) + | > loss_mel_ce: 3.0410308837890625 (3.438612467674387) + | > loss: 0.7691171169281006 (0.8703051111872541) + | > current_lr: 5e-06 + | > step_time: 0.28 (0.24523973888286577) + | > loader_time: 0.0117 (0.011741171580695849) + + + --> TIME: 2024-10-20 03:03:42 -- STEP: 39700/69059 -- GLOBAL_STEP: 39700 + | > loss_text_ce: 0.032987769693136215 (0.042597155875810054) + | > loss_mel_ce: 3.189940929412842 (3.4383154208954174) + | > loss: 0.8057321906089783 (0.870228144244224) + | > current_lr: 5e-06 + | > step_time: 0.1633 (0.24524239068667894) + | > loader_time: 0.0109 (0.01174087122045174) + + + --> TIME: 2024-10-20 03:04:00 -- STEP: 39750/69059 -- GLOBAL_STEP: 39750 + | > loss_text_ce: 0.03230126202106476 (0.042586417682843596) + | > loss_mel_ce: 3.1893672943115234 (3.4379939256763836) + | > loss: 0.8054171204566956 (0.870145085891089) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24524094639784136) + | > loader_time: 0.0103 (0.011740611346262784) + + + --> TIME: 2024-10-20 03:04:18 -- STEP: 39800/69059 -- GLOBAL_STEP: 39800 + | > loss_text_ce: 0.03424529731273651 (0.04257531933823408) + | > loss_mel_ce: 3.409562587738037 (3.4376635787654544) + | > loss: 0.8609519600868225 (0.8700597245744136) + | > current_lr: 5e-06 + | > step_time: 0.3917 (0.2452266900623264) + | > loader_time: 0.013 (0.011740211188493643) + + + --> TIME: 2024-10-20 03:04:36 -- STEP: 39850/69059 -- GLOBAL_STEP: 39850 + | > loss_text_ce: 0.03410038724541664 (0.042565248481125216) + | > loss_mel_ce: 3.2232420444488525 (3.437394280553291) + | > loss: 0.8143355846405029 (0.8699898823067426) + | > current_lr: 5e-06 + | > step_time: 0.2067 (0.24521686381647548) + | > loader_time: 0.0114 (0.011739711905064206) + + + --> TIME: 2024-10-20 03:04:55 -- STEP: 39900/69059 -- GLOBAL_STEP: 39900 + | > loss_text_ce: 0.03324886038899422 (0.042554750006160276) + | > loss_mel_ce: 3.749424934387207 (3.4371078252732388) + | > loss: 0.9456684589385986 (0.8699156438706822) + | > current_lr: 5e-06 + | > step_time: 0.2868 (0.24521962278767637) + | > loader_time: 0.0127 (0.01173909531500105) + + + --> TIME: 2024-10-20 03:05:12 -- STEP: 39950/69059 -- GLOBAL_STEP: 39950 + | > loss_text_ce: 0.03280617296695709 (0.04254420033044102) + | > loss_mel_ce: 3.213489294052124 (3.436793458202135) + | > loss: 0.81157386302948 (0.8698344146832369) + | > current_lr: 5e-06 + | > step_time: 0.3753 (0.24519853356185933) + | > loader_time: 0.012 (0.011738496166892872) + + + --> TIME: 2024-10-20 03:05:31 -- STEP: 40000/69059 -- GLOBAL_STEP: 40000 + | > loss_text_ce: 0.030402448028326035 (0.04253323433045282) + | > loss_mel_ce: 2.724120616912842 (3.43650421181915) + | > loss: 0.6886307597160339 (0.8697593615829956) + | > current_lr: 5e-06 + | > step_time: 0.1993 (0.2452053123652935) + | > loader_time: 0.0112 (0.01173818372488021) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_40000.pth + + --> TIME: 2024-10-20 03:06:03 -- STEP: 40050/69059 -- GLOBAL_STEP: 40050 + | > loss_text_ce: 0.03489700332283974 (0.042523017121425255) + | > loss_mel_ce: 3.1002137660980225 (3.436164312648394) + | > loss: 0.7837777137756348 (0.8696718324927366) + | > current_lr: 5e-06 + | > step_time: 0.2624 (0.24516666652855654) + | > loader_time: 0.0122 (0.01173786431811423) + + + --> TIME: 2024-10-20 03:06:21 -- STEP: 40100/69059 -- GLOBAL_STEP: 40100 + | > loss_text_ce: 0.03161928057670593 (0.042512770447080926) + | > loss_mel_ce: 2.9640541076660156 (3.4359058651483942) + | > loss: 0.748918354511261 (0.8696046589497033) + | > current_lr: 5e-06 + | > step_time: 0.1745 (0.2451576527990308) + | > loader_time: 0.0117 (0.011737777818170857) + + + --> TIME: 2024-10-20 03:06:40 -- STEP: 40150/69059 -- GLOBAL_STEP: 40150 + | > loss_text_ce: 0.03509960323572159 (0.04250255573578022) + | > loss_mel_ce: 3.0741965770721436 (3.435623644024743) + | > loss: 0.7773240208625793 (0.8695315499843143) + | > current_lr: 5e-06 + | > step_time: 0.2231 (0.24515891396390693) + | > loader_time: 0.0121 (0.011737959043471918) + + + --> TIME: 2024-10-20 03:06:58 -- STEP: 40200/69059 -- GLOBAL_STEP: 40200 + | > loss_text_ce: 0.03336404263973236 (0.04249239021328386) + | > loss_mel_ce: 2.915357828140259 (3.4353213614847906) + | > loss: 0.7371804714202881 (0.8694534379688672) + | > current_lr: 5e-06 + | > step_time: 0.181 (0.24514972230095175) + | > loader_time: 0.0111 (0.011737491212674029) + + + --> TIME: 2024-10-20 03:07:16 -- STEP: 40250/69059 -- GLOBAL_STEP: 40250 + | > loss_text_ce: 0.03494386747479439 (0.042482079964894504) + | > loss_mel_ce: 3.4405574798583984 (3.435055697168601) + | > loss: 0.8688753247261047 (0.8693844443241272) + | > current_lr: 5e-06 + | > step_time: 0.3246 (0.24515192150921555) + | > loader_time: 0.0129 (0.011737854418547246) + + + --> TIME: 2024-10-20 03:07:33 -- STEP: 40300/69059 -- GLOBAL_STEP: 40300 + | > loss_text_ce: 0.03750351071357727 (0.04247188785152661) + | > loss_mel_ce: 3.10579776763916 (3.4347626088156376) + | > loss: 0.7858253121376038 (0.8693086242039813) + | > current_lr: 5e-06 + | > step_time: 0.1627 (0.24511972504868992) + | > loader_time: 0.0106 (0.011737412974497215) + + + --> TIME: 2024-10-20 03:07:52 -- STEP: 40350/69059 -- GLOBAL_STEP: 40350 + | > loss_text_ce: 0.033453796058893204 (0.042461924764617145) + | > loss_mel_ce: 3.186634063720703 (3.4345128830007834) + | > loss: 0.8050219416618347 (0.869243701975586) + | > current_lr: 5e-06 + | > step_time: 0.1719 (0.2451159080224114) + | > loader_time: 0.0103 (0.011737348120451855) + + + --> TIME: 2024-10-20 03:08:11 -- STEP: 40400/69059 -- GLOBAL_STEP: 40400 + | > loss_text_ce: 0.03414513170719147 (0.04245164536617419) + | > loss_mel_ce: 3.053180694580078 (3.4342112705258825) + | > loss: 0.7718314528465271 (0.8691657290054441) + | > current_lr: 5e-06 + | > step_time: 0.1639 (0.2451311031839635) + | > loader_time: 0.0115 (0.011737390508746155) + + + --> TIME: 2024-10-20 03:08:31 -- STEP: 40450/69059 -- GLOBAL_STEP: 40450 + | > loss_text_ce: 0.0378715954720974 (0.04244172217612157) + | > loss_mel_ce: 3.1747357845306396 (3.433948551798022) + | > loss: 0.8031518459320068 (0.8690975685246364) + | > current_lr: 5e-06 + | > step_time: 0.3828 (0.24515775415305008) + | > loader_time: 0.013 (0.011738146539082185) + + + --> TIME: 2024-10-20 03:08:50 -- STEP: 40500/69059 -- GLOBAL_STEP: 40500 + | > loss_text_ce: 0.03554808348417282 (0.042431798168032435) + | > loss_mel_ce: 3.1252779960632324 (3.4336862223707385) + | > loss: 0.7902064919471741 (0.8690295051630643) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.2451660790031339) + | > loader_time: 0.011 (0.011738070605713637) + + + --> TIME: 2024-10-20 03:09:10 -- STEP: 40550/69059 -- GLOBAL_STEP: 40550 + | > loss_text_ce: 0.03477904573082924 (0.04242177211983818) + | > loss_mel_ce: 2.9487392902374268 (3.433477302444239) + | > loss: 0.745879590511322 (0.8689747686671281) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24518463145936903) + | > loader_time: 0.0103 (0.01173810684284063) + + + --> TIME: 2024-10-20 03:09:29 -- STEP: 40600/69059 -- GLOBAL_STEP: 40600 + | > loss_text_ce: 0.033214591443538666 (0.04241226328243234) + | > loss_mel_ce: 3.3297054767608643 (3.433239606795031) + | > loss: 0.8407300114631653 (0.8689129675403615) + | > current_lr: 5e-06 + | > step_time: 0.2683 (0.24520716614323884) + | > loader_time: 0.0128 (0.011737969456047836) + + + --> TIME: 2024-10-20 03:09:48 -- STEP: 40650/69059 -- GLOBAL_STEP: 40650 + | > loss_text_ce: 0.0354694165289402 (0.042402567811502596) + | > loss_mel_ce: 3.318485736846924 (3.4328953826793818) + | > loss: 0.8384888172149658 (0.8688244876401169) + | > current_lr: 5e-06 + | > step_time: 0.1722 (0.24520354124892682) + | > loader_time: 0.0107 (0.011737575372646648) + + + --> TIME: 2024-10-20 03:10:06 -- STEP: 40700/69059 -- GLOBAL_STEP: 40700 + | > loss_text_ce: 0.035205304622650146 (0.042392713755359905) + | > loss_mel_ce: 3.019662618637085 (3.4325645457965814) + | > loss: 0.763716995716095 (0.8687393149067212) + | > current_lr: 5e-06 + | > step_time: 0.3115 (0.24520209296036705) + | > loader_time: 0.0122 (0.011737189140600996) + + + --> TIME: 2024-10-20 03:10:25 -- STEP: 40750/69059 -- GLOBAL_STEP: 40750 + | > loss_text_ce: 0.03645366057753563 (0.0423827964919932) + | > loss_mel_ce: 3.4056084156036377 (3.432320679600232) + | > loss: 0.8605155348777771 (0.8686758690392323) + | > current_lr: 5e-06 + | > step_time: 0.232 (0.245209695915503) + | > loader_time: 0.0114 (0.011737150051842428) + + + --> TIME: 2024-10-20 03:10:44 -- STEP: 40800/69059 -- GLOBAL_STEP: 40800 + | > loss_text_ce: 0.03442865237593651 (0.04237331696265524) + | > loss_mel_ce: 3.490610122680664 (3.432040012583992) + | > loss: 0.8812596797943115 (0.8686033324049979) + | > current_lr: 5e-06 + | > step_time: 0.2465 (0.24521199830022514) + | > loader_time: 0.0119 (0.01173711083098953) + + + --> TIME: 2024-10-20 03:11:01 -- STEP: 40850/69059 -- GLOBAL_STEP: 40850 + | > loss_text_ce: 0.03701937943696976 (0.042363020974317205) + | > loss_mel_ce: 3.3388657569885254 (3.431670071317028) + | > loss: 0.843971312046051 (0.86850827309172) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.24518174530972467) + | > loader_time: 0.0102 (0.011736287154357603) + + + --> TIME: 2024-10-20 03:11:18 -- STEP: 40900/69059 -- GLOBAL_STEP: 40900 + | > loss_text_ce: 0.033833861351013184 (0.04235206002160007) + | > loss_mel_ce: 3.190556049346924 (3.4313573724074953) + | > loss: 0.8060975074768066 (0.8684273581268745) + | > current_lr: 5e-06 + | > step_time: 0.1523 (0.24516637184217682) + | > loader_time: 0.0108 (0.011735776286836343) + + + --> TIME: 2024-10-20 03:11:36 -- STEP: 40950/69059 -- GLOBAL_STEP: 40950 + | > loss_text_ce: 0.03252972662448883 (0.042342254947711475) + | > loss_mel_ce: 3.498745918273926 (3.4311085427986185) + | > loss: 0.8828189373016357 (0.868362699447244) + | > current_lr: 5e-06 + | > step_time: 0.2109 (0.2451510464490115) + | > loader_time: 0.0106 (0.01173548055655788) + + + --> TIME: 2024-10-20 03:11:55 -- STEP: 41000/69059 -- GLOBAL_STEP: 41000 + | > loss_text_ce: 0.03504331782460213 (0.042332210923749204) + | > loss_mel_ce: 3.601742744445801 (3.4308056110172522) + | > loss: 0.9091964960098267 (0.8682844554970914) + | > current_lr: 5e-06 + | > step_time: 0.1677 (0.24514787106397676) + | > loader_time: 0.0143 (0.011735044996912865) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_41000.pth + + --> TIME: 2024-10-20 03:12:28 -- STEP: 41050/69059 -- GLOBAL_STEP: 41050 + | > loss_text_ce: 0.03510934114456177 (0.042322092841852114) + | > loss_mel_ce: 3.09983491897583 (3.430516440354348) + | > loss: 0.7837360501289368 (0.8682096333044902) + | > current_lr: 5e-06 + | > step_time: 0.207 (0.24514933814375178) + | > loader_time: 0.0105 (0.011735031706406898) + + + --> TIME: 2024-10-20 03:12:46 -- STEP: 41100/69059 -- GLOBAL_STEP: 41100 + | > loss_text_ce: 0.0365833044052124 (0.04231265525190799) + | > loss_mel_ce: 3.1237926483154297 (3.4302294621502183) + | > loss: 0.7900940179824829 (0.8681355293548321) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.245137500994977) + | > loader_time: 0.0122 (0.01173487043728792) + + + --> TIME: 2024-10-20 03:13:03 -- STEP: 41150/69059 -- GLOBAL_STEP: 41150 + | > loss_text_ce: 0.03405998274683952 (0.042302210595082775) + | > loss_mel_ce: 3.2385995388031006 (3.4298946761795053) + | > loss: 0.8181648850440979 (0.8680492216978745) + | > current_lr: 5e-06 + | > step_time: 0.1718 (0.24511044136084445) + | > loader_time: 0.0109 (0.01173421703885565) + + + --> TIME: 2024-10-20 03:13:21 -- STEP: 41200/69059 -- GLOBAL_STEP: 41200 + | > loss_text_ce: 0.03247779607772827 (0.042292131079559495) + | > loss_mel_ce: 3.0901691913604736 (3.4296287902176745) + | > loss: 0.7806617617607117 (0.8679802303291072) + | > current_lr: 5e-06 + | > step_time: 0.2276 (0.24509545579118636) + | > loader_time: 0.0115 (0.011733975618788325) + + + --> TIME: 2024-10-20 03:13:40 -- STEP: 41250/69059 -- GLOBAL_STEP: 41250 + | > loss_text_ce: 0.03508852422237396 (0.042282674858696245) + | > loss_mel_ce: 3.576983690261841 (3.4293701077547727) + | > loss: 0.903018057346344 (0.8679131956591761) + | > current_lr: 5e-06 + | > step_time: 0.3675 (0.24511008386322947) + | > loader_time: 0.013 (0.011733874072450565) + + + --> TIME: 2024-10-20 03:13:58 -- STEP: 41300/69059 -- GLOBAL_STEP: 41300 + | > loss_text_ce: 0.03317873179912567 (0.042272588722407536) + | > loss_mel_ce: 3.2449326515197754 (3.429095802930575) + | > loss: 0.8195278644561768 (0.8678420979217535) + | > current_lr: 5e-06 + | > step_time: 0.2689 (0.24509170447942996) + | > loader_time: 0.0122 (0.011733434015555752) + + + --> TIME: 2024-10-20 03:14:18 -- STEP: 41350/69059 -- GLOBAL_STEP: 41350 + | > loss_text_ce: 0.03999551758170128 (0.04226323417769634) + | > loss_mel_ce: 3.4033617973327637 (3.4288944446016907) + | > loss: 0.860839307308197 (0.8677894197020605) + | > current_lr: 5e-06 + | > step_time: 0.425 (0.24512322638602757) + | > loader_time: 0.0138 (0.011733521238924361) + + + --> TIME: 2024-10-20 03:14:37 -- STEP: 41400/69059 -- GLOBAL_STEP: 41400 + | > loss_text_ce: 0.033363670110702515 (0.042253679218409265) + | > loss_mel_ce: 2.930612087249756 (3.428614599842935) + | > loss: 0.7409939169883728 (0.8677170697718437) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.24512708154853416) + | > loader_time: 0.0106 (0.01173626235717736) + + + --> TIME: 2024-10-20 03:14:55 -- STEP: 41450/69059 -- GLOBAL_STEP: 41450 + | > loss_text_ce: 0.03785653039813042 (0.042243967687101924) + | > loss_mel_ce: 3.0746235847473145 (3.4283211764610364) + | > loss: 0.7781200408935547 (0.8676412860360748) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.24512718924279667) + | > loader_time: 0.011 (0.011735932041853834) + + + --> TIME: 2024-10-20 03:15:15 -- STEP: 41500/69059 -- GLOBAL_STEP: 41500 + | > loss_text_ce: 0.03494623303413391 (0.04223397294871748) + | > loss_mel_ce: 3.105893850326538 (3.428025733149174) + | > loss: 0.7852100133895874 (0.8675649265180162) + | > current_lr: 5e-06 + | > step_time: 0.2427 (0.24514155320661613) + | > loader_time: 0.0115 (0.011735520368599017) + + + --> TIME: 2024-10-20 03:15:34 -- STEP: 41550/69059 -- GLOBAL_STEP: 41550 + | > loss_text_ce: 0.0349101647734642 (0.042224791003021026) + | > loss_mel_ce: 3.4432547092437744 (3.427784903476937) + | > loss: 0.8695412278175354 (0.8675024236109293) + | > current_lr: 5e-06 + | > step_time: 0.4177 (0.2451537830763632) + | > loader_time: 0.013 (0.011735249380485806) + + + --> TIME: 2024-10-20 03:15:52 -- STEP: 41600/69059 -- GLOBAL_STEP: 41600 + | > loss_text_ce: 0.03733932226896286 (0.04221486228285328) + | > loss_mel_ce: 3.362767219543457 (3.4274965981737715) + | > loss: 0.8500266075134277 (0.867427865106327) + | > current_lr: 5e-06 + | > step_time: 0.3945 (0.24513475915560357) + | > loader_time: 0.0127 (0.011734827733956836) + + + --> TIME: 2024-10-20 03:16:09 -- STEP: 41650/69059 -- GLOBAL_STEP: 41650 + | > loss_text_ce: 0.032557833939790726 (0.04220484057863941) + | > loss_mel_ce: 3.2895565032958984 (3.4272498972103738) + | > loss: 0.8305285573005676 (0.8673636844349948) + | > current_lr: 5e-06 + | > step_time: 0.2616 (0.24512143566113273) + | > loader_time: 0.0118 (0.011734171423162153) + + + --> TIME: 2024-10-20 03:16:28 -- STEP: 41700/69059 -- GLOBAL_STEP: 41700 + | > loss_text_ce: 0.035847876220941544 (0.042195148348352376) + | > loss_mel_ce: 3.4244580268859863 (3.426951418511852) + | > loss: 0.86507648229599 (0.8672866417049514) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24512082135934624) + | > loader_time: 0.0118 (0.011733947217607375) + + + --> TIME: 2024-10-20 03:16:46 -- STEP: 41750/69059 -- GLOBAL_STEP: 41750 + | > loss_text_ce: 0.032345373183488846 (0.04218560375902259) + | > loss_mel_ce: 3.1509828567504883 (3.4267357341457774) + | > loss: 0.7958320379257202 (0.8672303344660901) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24511459046209644) + | > loader_time: 0.0108 (0.011734013300455958) + + + --> TIME: 2024-10-20 03:17:07 -- STEP: 41800/69059 -- GLOBAL_STEP: 41800 + | > loss_text_ce: 0.031986624002456665 (0.042175751161676614) + | > loss_mel_ce: 3.4308998584747314 (3.4264817618696104) + | > loss: 0.8657216429710388 (0.8671643782460529) + | > current_lr: 5e-06 + | > step_time: 0.296 (0.2451533714207736) + | > loader_time: 0.0132 (0.011734030674519145) + + + --> TIME: 2024-10-20 03:17:25 -- STEP: 41850/69059 -- GLOBAL_STEP: 41850 + | > loss_text_ce: 0.03333522379398346 (0.04216619647865731) + | > loss_mel_ce: 3.5483269691467285 (3.42624615862376) + | > loss: 0.8954155445098877 (0.8671030887665867) + | > current_lr: 5e-06 + | > step_time: 0.2314 (0.24514932430773226) + | > loader_time: 0.0106 (0.011734000015714539) + + + --> TIME: 2024-10-20 03:17:44 -- STEP: 41900/69059 -- GLOBAL_STEP: 41900 + | > loss_text_ce: 0.03435640037059784 (0.042156212083897186) + | > loss_mel_ce: 3.031857967376709 (3.425979303932397) + | > loss: 0.7665535807609558 (0.867033878998712) + | > current_lr: 5e-06 + | > step_time: 0.1522 (0.24514319427827094) + | > loader_time: 0.0105 (0.011733806087748802) + + + --> TIME: 2024-10-20 03:18:02 -- STEP: 41950/69059 -- GLOBAL_STEP: 41950 + | > loss_text_ce: 0.037785958498716354 (0.042147186176942054) + | > loss_mel_ce: 3.4651198387145996 (3.4257709479928518) + | > loss: 0.8757264614105225 (0.8669795335376372) + | > current_lr: 5e-06 + | > step_time: 0.3444 (0.24513831380155288) + | > loader_time: 0.013 (0.011733374612692266) + + + --> TIME: 2024-10-20 03:18:19 -- STEP: 42000/69059 -- GLOBAL_STEP: 42000 + | > loss_text_ce: 0.03310154750943184 (0.042137615304085094) + | > loss_mel_ce: 3.40274715423584 (3.4254681440932204) + | > loss: 0.8589621782302856 (0.8669014398413055) + | > current_lr: 5e-06 + | > step_time: 0.1638 (0.24512072480292546) + | > loader_time: 0.0106 (0.011732573242414546) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_42000.pth + + --> TIME: 2024-10-20 03:18:53 -- STEP: 42050/69059 -- GLOBAL_STEP: 42050 + | > loss_text_ce: 0.0345122404396534 (0.04212864393857795) + | > loss_mel_ce: 3.013817310333252 (3.425217224221997) + | > loss: 0.7620823979377747 (0.8668364670316342) + | > current_lr: 5e-06 + | > step_time: 0.1853 (0.24513861357386132) + | > loader_time: 0.0108 (0.011732458025039852) + + + --> TIME: 2024-10-20 03:19:11 -- STEP: 42100/69059 -- GLOBAL_STEP: 42100 + | > loss_text_ce: 0.036244988441467285 (0.04211938059329065) + | > loss_mel_ce: 3.0339906215667725 (3.4249213310205247) + | > loss: 0.7675589323043823 (0.8667601778954632) + | > current_lr: 5e-06 + | > step_time: 0.1653 (0.2451289169012509) + | > loader_time: 0.0106 (0.011732011696505127) + + + --> TIME: 2024-10-20 03:19:29 -- STEP: 42150/69059 -- GLOBAL_STEP: 42150 + | > loss_text_ce: 0.030231580138206482 (0.04211001503965114) + | > loss_mel_ce: 2.8934764862060547 (3.4245889357513457) + | > loss: 0.7309269905090332 (0.866674737695557) + | > current_lr: 5e-06 + | > step_time: 0.1718 (0.2451085901599762) + | > loader_time: 0.0106 (0.011731473084446352) + + + --> TIME: 2024-10-20 03:19:48 -- STEP: 42200/69059 -- GLOBAL_STEP: 42200 + | > loss_text_ce: 0.03129434585571289 (0.04210048645823029) + | > loss_mel_ce: 3.027348041534424 (3.42432127693816) + | > loss: 0.7646605968475342 (0.8666054408442928) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.2451183023384962) + | > loader_time: 0.011 (0.011731445280861497) + + + --> TIME: 2024-10-20 03:20:08 -- STEP: 42250/69059 -- GLOBAL_STEP: 42250 + | > loss_text_ce: 0.03545895591378212 (0.04209151794011916) + | > loss_mel_ce: 3.1613340377807617 (3.424093897097189) + | > loss: 0.7991982698440552 (0.8665463537549136) + | > current_lr: 5e-06 + | > step_time: 0.4867 (0.24514652474533172) + | > loader_time: 0.0136 (0.01173176383972166) + + + --> TIME: 2024-10-20 03:20:26 -- STEP: 42300/69059 -- GLOBAL_STEP: 42300 + | > loss_text_ce: 0.035947490483522415 (0.04208216805539408) + | > loss_mel_ce: 3.2212061882019043 (3.423829753387602) + | > loss: 0.8142884373664856 (0.8664779803586636) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24513475480654562) + | > loader_time: 0.0111 (0.011731481636669603) + + + --> TIME: 2024-10-20 03:20:45 -- STEP: 42350/69059 -- GLOBAL_STEP: 42350 + | > loss_text_ce: 0.03224838525056839 (0.04207272248692256) + | > loss_mel_ce: 3.10496187210083 (3.4236351809675964) + | > loss: 0.7843025922775269 (0.8664269758621955) + | > current_lr: 5e-06 + | > step_time: 0.1723 (0.2451505456370474) + | > loader_time: 0.0105 (0.011731617656198824) + + + --> TIME: 2024-10-20 03:21:05 -- STEP: 42400/69059 -- GLOBAL_STEP: 42400 + | > loss_text_ce: 0.034268058836460114 (0.042063405308257064) + | > loss_mel_ce: 3.1798717975616455 (3.4234035721252343) + | > loss: 0.803534984588623 (0.8663667443563363) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24517266108179991) + | > loader_time: 0.0107 (0.011731566528104373) + + + --> TIME: 2024-10-20 03:21:23 -- STEP: 42450/69059 -- GLOBAL_STEP: 42450 + | > loss_text_ce: 0.03466032072901726 (0.0420543420744356) + | > loss_mel_ce: 2.9835317134857178 (3.4231884121866547) + | > loss: 0.7545480132102966 (0.8663106885605071) + | > current_lr: 5e-06 + | > step_time: 0.2317 (0.2451669374090763) + | > loader_time: 0.0115 (0.011731450206680191) + + + --> TIME: 2024-10-20 03:21:43 -- STEP: 42500/69059 -- GLOBAL_STEP: 42500 + | > loss_text_ce: 0.03445916250348091 (0.04204483401415936) + | > loss_mel_ce: 3.051386833190918 (3.422920397623827) + | > loss: 0.7714614868164062 (0.866241307907947) + | > current_lr: 5e-06 + | > step_time: 0.182 (0.24519073215372422) + | > loader_time: 0.0105 (0.011731429661021494) + + + --> TIME: 2024-10-20 03:22:00 -- STEP: 42550/69059 -- GLOBAL_STEP: 42550 + | > loss_text_ce: 0.036275263875722885 (0.0420354397004394) + | > loss_mel_ce: 3.11138653755188 (3.4226557881627397) + | > loss: 0.7869154214859009 (0.8661728069602963) + | > current_lr: 5e-06 + | > step_time: 0.1872 (0.2451697511482463) + | > loader_time: 0.011 (0.011730970103648219) + + + --> TIME: 2024-10-20 03:22:17 -- STEP: 42600/69059 -- GLOBAL_STEP: 42600 + | > loss_text_ce: 0.03585027903318405 (0.04202550086135905) + | > loss_mel_ce: 3.4017388820648193 (3.422325370809821) + | > loss: 0.859397292137146 (0.8660877179078104) + | > current_lr: 5e-06 + | > step_time: 0.285 (0.24514496060044552) + | > loader_time: 0.0117 (0.011730273206469022) + + + --> TIME: 2024-10-20 03:22:35 -- STEP: 42650/69059 -- GLOBAL_STEP: 42650 + | > loss_text_ce: 0.03605308756232262 (0.04201572320879271) + | > loss_mel_ce: 3.229118585586548 (3.422028469731891) + | > loss: 0.816292941570282 (0.866011048219968) + | > current_lr: 5e-06 + | > step_time: 0.2081 (0.2451384950718315) + | > loader_time: 0.0145 (0.011730127619690788) + + + --> TIME: 2024-10-20 03:22:54 -- STEP: 42700/69059 -- GLOBAL_STEP: 42700 + | > loss_text_ce: 0.03432481363415718 (0.04200668671131269) + | > loss_mel_ce: 3.174769878387451 (3.4217588912742554) + | > loss: 0.802273690700531 (0.8659413944767576) + | > current_lr: 5e-06 + | > step_time: 0.2443 (0.24514563471148668) + | > loader_time: 0.0119 (0.011729835592890772) + + + --> TIME: 2024-10-20 03:23:13 -- STEP: 42750/69059 -- GLOBAL_STEP: 42750 + | > loss_text_ce: 0.03179670125246048 (0.04199710274470265) + | > loss_mel_ce: 3.0050313472747803 (3.421557760115932) + | > loss: 0.759207010269165 (0.8658887156966147) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.2451497518082111) + | > loader_time: 0.0131 (0.01172969766806438) + + + --> TIME: 2024-10-20 03:23:32 -- STEP: 42800/69059 -- GLOBAL_STEP: 42800 + | > loss_text_ce: 0.0347648523747921 (0.04198751913235594) + | > loss_mel_ce: 3.4059417247772217 (3.421299056538891) + | > loss: 0.8601766228675842 (0.8658216439006495) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.2451469697239243) + | > loader_time: 0.0119 (0.011729563157135053) + + + --> TIME: 2024-10-20 03:23:50 -- STEP: 42850/69059 -- GLOBAL_STEP: 42850 + | > loss_text_ce: 0.033721163868904114 (0.0419780177683886) + | > loss_mel_ce: 2.8468780517578125 (3.420981513972579) + | > loss: 0.72014981508255 (0.8657398829173607) + | > current_lr: 5e-06 + | > step_time: 0.1944 (0.24513976455986708) + | > loader_time: 0.0106 (0.011729603214330656) + + + --> TIME: 2024-10-20 03:24:10 -- STEP: 42900/69059 -- GLOBAL_STEP: 42900 + | > loss_text_ce: 0.033709824085235596 (0.04196880977392363) + | > loss_mel_ce: 3.1555182933807373 (3.4207556727493635) + | > loss: 0.797307014465332 (0.8656811206101825) + | > current_lr: 5e-06 + | > step_time: 0.3858 (0.24517121132715042) + | > loader_time: 0.0138 (0.011729737272907213) + + + --> TIME: 2024-10-20 03:24:28 -- STEP: 42950/69059 -- GLOBAL_STEP: 42950 + | > loss_text_ce: 0.033544719219207764 (0.04195916423558868) + | > loss_mel_ce: 3.3901238441467285 (3.4204724503562467) + | > loss: 0.8559171557426453 (0.8656079036254138) + | > current_lr: 5e-06 + | > step_time: 0.2626 (0.24516206911750946) + | > loader_time: 0.0112 (0.011729225551707332) + + + --> TIME: 2024-10-20 03:24:48 -- STEP: 43000/69059 -- GLOBAL_STEP: 43000 + | > loss_text_ce: 0.03413684293627739 (0.0419505069064362) + | > loss_mel_ce: 3.2214789390563965 (3.4202571305618528) + | > loss: 0.8139039278030396 (0.8655519093413696) + | > current_lr: 5e-06 + | > step_time: 0.1653 (0.24519094831998958) + | > loader_time: 0.011 (0.011729216098785364) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_43000.pth + + --> TIME: 2024-10-20 03:25:20 -- STEP: 43050/69059 -- GLOBAL_STEP: 43050 + | > loss_text_ce: 0.032289378345012665 (0.041941692319446174) + | > loss_mel_ce: 2.9598424434661865 (3.4200018921961566) + | > loss: 0.7480329275131226 (0.8654858961019515) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.2451814278769853) + | > loader_time: 0.0109 (0.011729189532697517) + + + --> TIME: 2024-10-20 03:25:39 -- STEP: 43100/69059 -- GLOBAL_STEP: 43100 + | > loss_text_ce: 0.034387730062007904 (0.0419325069145764) + | > loss_mel_ce: 3.271012783050537 (3.419771542311248) + | > loss: 0.8263501524925232 (0.865426012281753) + | > current_lr: 5e-06 + | > step_time: 0.1973 (0.24519122996628975) + | > loader_time: 0.0112 (0.011729342118769928) + + + --> TIME: 2024-10-20 03:25:58 -- STEP: 43150/69059 -- GLOBAL_STEP: 43150 + | > loss_text_ce: 0.034484751522541046 (0.04192338401673973) + | > loss_mel_ce: 3.3035473823547363 (3.4195823680097304) + | > loss: 0.8345080614089966 (0.8653764379798063) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.2452093344906114) + | > loader_time: 0.0106 (0.011729740919685563) + + + --> TIME: 2024-10-20 03:26:16 -- STEP: 43200/69059 -- GLOBAL_STEP: 43200 + | > loss_text_ce: 0.036538101732730865 (0.04191389646803645) + | > loss_mel_ce: 3.0305068492889404 (3.419322152463354) + | > loss: 0.7667612433433533 (0.8653090122052375) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.2451976434941645) + | > loader_time: 0.0122 (0.011729557894998102) + + + --> TIME: 2024-10-20 03:26:34 -- STEP: 43250/69059 -- GLOBAL_STEP: 43250 + | > loss_text_ce: 0.0340290404856205 (0.0419048725182262) + | > loss_mel_ce: 2.910945415496826 (3.4190707332522683) + | > loss: 0.7362436056137085 (0.8652439014153683) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.2451814701157498) + | > loader_time: 0.012 (0.011729731438476885) + + + --> TIME: 2024-10-20 03:26:52 -- STEP: 43300/69059 -- GLOBAL_STEP: 43300 + | > loss_text_ce: 0.03168216720223427 (0.04189559946279701) + | > loss_mel_ce: 3.0033514499664307 (3.4187661132525977) + | > loss: 0.7587584257125854 (0.8651654281544641) + | > current_lr: 5e-06 + | > step_time: 0.1809 (0.2451699087856548) + | > loader_time: 0.011 (0.01173017693997528) + + + --> TIME: 2024-10-20 03:27:12 -- STEP: 43350/69059 -- GLOBAL_STEP: 43350 + | > loss_text_ce: 0.03442641347646713 (0.04188668364704028) + | > loss_mel_ce: 3.108941078186035 (3.4186008235354812) + | > loss: 0.7858418822288513 (0.8651218767713383) + | > current_lr: 5e-06 + | > step_time: 0.4072 (0.24520137328177588) + | > loader_time: 0.0139 (0.011730859738870175) + + + --> TIME: 2024-10-20 03:27:30 -- STEP: 43400/69059 -- GLOBAL_STEP: 43400 + | > loss_text_ce: 0.03342118114233017 (0.041877465135318766) + | > loss_mel_ce: 3.0840933322906494 (3.418288412390746) + | > loss: 0.7793786525726318 (0.8650414693589045) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.24519478947336223) + | > loader_time: 0.0111 (0.011730682866364552) + + + --> TIME: 2024-10-20 03:27:48 -- STEP: 43450/69059 -- GLOBAL_STEP: 43450 + | > loss_text_ce: 0.0347004160284996 (0.041868757806092054) + | > loss_mel_ce: 3.362403631210327 (3.4180407105271167) + | > loss: 0.8492760062217712 (0.8649773670587232) + | > current_lr: 5e-06 + | > step_time: 0.2592 (0.24517733555257115) + | > loader_time: 0.0117 (0.011730609576638986) + + + --> TIME: 2024-10-20 03:28:06 -- STEP: 43500/69059 -- GLOBAL_STEP: 43500 + | > loss_text_ce: 0.030881276354193687 (0.041860145128363765) + | > loss_mel_ce: 3.2318596839904785 (3.4177486745516257) + | > loss: 0.8156852126121521 (0.8649022048931022) + | > current_lr: 5e-06 + | > step_time: 0.1571 (0.24517426608622758) + | > loader_time: 0.0104 (0.01173048281943658) + + + --> TIME: 2024-10-20 03:28:25 -- STEP: 43550/69059 -- GLOBAL_STEP: 43550 + | > loss_text_ce: 0.03529255464673042 (0.04185114058985597) + | > loss_mel_ce: 3.254807472229004 (3.417511164936742) + | > loss: 0.8225250244140625 (0.8648405763558223) + | > current_lr: 5e-06 + | > step_time: 0.2158 (0.24517616755379326) + | > loader_time: 0.0119 (0.01173044950351647) + + + --> TIME: 2024-10-20 03:28:45 -- STEP: 43600/69059 -- GLOBAL_STEP: 43600 + | > loss_text_ce: 0.03272634372115135 (0.04184240246084776) + | > loss_mel_ce: 3.0250730514526367 (3.417264439906529) + | > loss: 0.7644498348236084 (0.8647767105671254) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.24519972794099684) + | > loader_time: 0.0112 (0.01173066996106311) + + + --> TIME: 2024-10-20 03:29:04 -- STEP: 43650/69059 -- GLOBAL_STEP: 43650 + | > loss_text_ce: 0.03623170778155327 (0.04183357688872461) + | > loss_mel_ce: 3.3515443801879883 (3.417052927541549) + | > loss: 0.8469440340995789 (0.8647216260815551) + | > current_lr: 5e-06 + | > step_time: 0.4483 (0.24520734115154882) + | > loader_time: 0.0133 (0.011730571727162757) + + + --> TIME: 2024-10-20 03:29:23 -- STEP: 43700/69059 -- GLOBAL_STEP: 43700 + | > loss_text_ce: 0.03773632273077965 (0.04182432915555009) + | > loss_mel_ce: 3.2683961391448975 (3.4168306590381223) + | > loss: 0.8265331387519836 (0.8646637470198452) + | > current_lr: 5e-06 + | > step_time: 0.247 (0.24522254361713505) + | > loader_time: 0.0112 (0.011730548326156207) + + + --> TIME: 2024-10-20 03:29:41 -- STEP: 43750/69059 -- GLOBAL_STEP: 43750 + | > loss_text_ce: 0.03708174079656601 (0.04181514367610172) + | > loss_mel_ce: 3.3524329662323 (3.4165706649126126) + | > loss: 0.847378671169281 (0.8645964521108365) + | > current_lr: 5e-06 + | > step_time: 0.4235 (0.24522018522535052) + | > loader_time: 0.013 (0.011730094947814915) + + + --> TIME: 2024-10-20 03:29:59 -- STEP: 43800/69059 -- GLOBAL_STEP: 43800 + | > loss_text_ce: 0.03521158546209335 (0.041806894859222114) + | > loss_mel_ce: 3.4457547664642334 (3.416288895280374) + | > loss: 0.8702415823936462 (0.8645239474974821) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.24520173269319753) + | > loader_time: 0.0106 (0.011729592177421509) + + + --> TIME: 2024-10-20 03:30:18 -- STEP: 43850/69059 -- GLOBAL_STEP: 43850 + | > loss_text_ce: 0.03843303397297859 (0.04179751509152729) + | > loss_mel_ce: 3.333737373352051 (3.4160529045259196) + | > loss: 0.8430426120758057 (0.864462604861168) + | > current_lr: 5e-06 + | > step_time: 0.2681 (0.24520462278629246) + | > loader_time: 0.0119 (0.011729348980850316) + + + --> TIME: 2024-10-20 03:30:35 -- STEP: 43900/69059 -- GLOBAL_STEP: 43900 + | > loss_text_ce: 0.03493482619524002 (0.04178847440745394) + | > loss_mel_ce: 2.9903039932250977 (3.415793173209933) + | > loss: 0.7563096880912781 (0.8643954118631434) + | > current_lr: 5e-06 + | > step_time: 0.157 (0.24518284806357973) + | > loader_time: 0.0114 (0.011728767338537656) + + + --> TIME: 2024-10-20 03:30:53 -- STEP: 43950/69059 -- GLOBAL_STEP: 43950 + | > loss_text_ce: 0.038651205599308014 (0.04177978971424186) + | > loss_mel_ce: 3.10551381111145 (3.4155340389885125) + | > loss: 0.786041259765625 (0.8643284571347122) + | > current_lr: 5e-06 + | > step_time: 0.1878 (0.2451792032482681) + | > loader_time: 0.0137 (0.011728475356942804) + + + --> TIME: 2024-10-20 03:31:13 -- STEP: 44000/69059 -- GLOBAL_STEP: 44000 + | > loss_text_ce: 0.033458162099123 (0.041771232640285635) + | > loss_mel_ce: 3.0416767597198486 (3.4152889169725307) + | > loss: 0.7687837481498718 (0.8642650373605174) + | > current_lr: 5e-06 + | > step_time: 0.1519 (0.24519885579022493) + | > loader_time: 0.0101 (0.011728319103067545) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_44000.pth + + --> TIME: 2024-10-20 03:31:46 -- STEP: 44050/69059 -- GLOBAL_STEP: 44050 + | > loss_text_ce: 0.03135424852371216 (0.04176232113571589) + | > loss_mel_ce: 3.27647066116333 (3.414965306210579) + | > loss: 0.8269562125205994 (0.8641819067967476) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.24519036031067032) + | > loader_time: 0.0112 (0.011728132179727863) + + + --> TIME: 2024-10-20 03:32:06 -- STEP: 44100/69059 -- GLOBAL_STEP: 44100 + | > loss_text_ce: 0.03415266051888466 (0.04175389760581396) + | > loss_mel_ce: 3.5505411624908447 (3.4147362641249996) + | > loss: 0.8961734771728516 (0.8641225403979802) + | > current_lr: 5e-06 + | > step_time: 0.242 (0.2452028547014509) + | > loader_time: 0.0123 (0.011728096170490265) + + + --> TIME: 2024-10-20 03:32:24 -- STEP: 44150/69059 -- GLOBAL_STEP: 44150 + | > loss_text_ce: 0.03483722731471062 (0.04174489987902252) + | > loss_mel_ce: 3.483644962310791 (3.4144422186865344) + | > loss: 0.8796205520629883 (0.8640467796063784) + | > current_lr: 5e-06 + | > step_time: 0.2359 (0.24519904098856354) + | > loader_time: 0.0112 (0.011727908086938714) + + + --> TIME: 2024-10-20 03:32:43 -- STEP: 44200/69059 -- GLOBAL_STEP: 44200 + | > loss_text_ce: 0.0338895320892334 (0.04173615773275815) + | > loss_mel_ce: 2.88372540473938 (3.414169651205164) + | > loss: 0.7294037342071533 (0.8639764522009316) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.24520867220956277) + | > loader_time: 0.0106 (0.011727965163968781) + + + --> TIME: 2024-10-20 03:33:03 -- STEP: 44250/69059 -- GLOBAL_STEP: 44250 + | > loss_text_ce: 0.03447249159216881 (0.04172775834281007) + | > loss_mel_ce: 3.0829498767852783 (3.413984684475378) + | > loss: 0.7793555855751038 (0.8639281106722567) + | > current_lr: 5e-06 + | > step_time: 0.3137 (0.24523395456971422) + | > loader_time: 0.0128 (0.011728298192643824) + + + --> TIME: 2024-10-20 03:33:22 -- STEP: 44300/69059 -- GLOBAL_STEP: 44300 + | > loss_text_ce: 0.03377753496170044 (0.04171862927835134) + | > loss_mel_ce: 3.2110955715179443 (3.4137299339227503) + | > loss: 0.81121826171875 (0.863862140769346) + | > current_lr: 5e-06 + | > step_time: 0.4275 (0.24523206458403887) + | > loader_time: 0.0136 (0.011728017437807837) + + + --> TIME: 2024-10-20 03:33:40 -- STEP: 44350/69059 -- GLOBAL_STEP: 44350 + | > loss_text_ce: 0.036701057106256485 (0.04171008036489696) + | > loss_mel_ce: 3.0656371116638184 (3.4134452244716478) + | > loss: 0.7755845189094543 (0.8637888261766562) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24522606713124032) + | > loader_time: 0.0113 (0.011727933378337847) + + + --> TIME: 2024-10-20 03:33:58 -- STEP: 44400/69059 -- GLOBAL_STEP: 44400 + | > loss_text_ce: 0.03440666198730469 (0.0417010127960254) + | > loss_mel_ce: 3.3853189945220947 (3.413175730264919) + | > loss: 0.8549314141273499 (0.8637191857357293) + | > current_lr: 5e-06 + | > step_time: 0.2038 (0.2452083856243271) + | > loader_time: 0.0115 (0.011727788050969416) + + + --> TIME: 2024-10-20 03:34:16 -- STEP: 44450/69059 -- GLOBAL_STEP: 44450 + | > loss_text_ce: 0.038828231394290924 (0.04169238201146566) + | > loss_mel_ce: 3.1007308959960938 (3.4128970939155443) + | > loss: 0.7848897576332092 (0.8636473689505713) + | > current_lr: 5e-06 + | > step_time: 0.2323 (0.24520983908924665) + | > loader_time: 0.0117 (0.011727803496327498) + + + --> TIME: 2024-10-20 03:34:35 -- STEP: 44500/69059 -- GLOBAL_STEP: 44500 + | > loss_text_ce: 0.03584470599889755 (0.04168380266289813) + | > loss_mel_ce: 3.5308761596679688 (3.4126479112324892) + | > loss: 0.8916802406311035 (0.8635829284418844) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.2452153405821725) + | > loader_time: 0.0106 (0.011727636460507828) + + + --> TIME: 2024-10-20 03:34:54 -- STEP: 44550/69059 -- GLOBAL_STEP: 44550 + | > loss_text_ce: 0.03328809514641762 (0.04167553028587188) + | > loss_mel_ce: 3.051889181137085 (3.412408668740678) + | > loss: 0.7712942957878113 (0.863521049727763) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.2452129905231874) + | > loader_time: 0.0114 (0.011727386430725345) + + + --> TIME: 2024-10-20 03:35:11 -- STEP: 44600/69059 -- GLOBAL_STEP: 44600 + | > loss_text_ce: 0.03354193642735481 (0.04166681216921057) + | > loss_mel_ce: 3.0228848457336426 (3.4121135309405197) + | > loss: 0.7641066908836365 (0.8634450857508352) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.24519094558574694) + | > loader_time: 0.0104 (0.011726770326161038) + + + --> TIME: 2024-10-20 03:35:31 -- STEP: 44650/69059 -- GLOBAL_STEP: 44650 + | > loss_text_ce: 0.03646068274974823 (0.04165825382834472) + | > loss_mel_ce: 2.998973846435547 (3.4119000195442166) + | > loss: 0.7588586211204529 (0.8633895683181789) + | > current_lr: 5e-06 + | > step_time: 0.211 (0.24521289777381958) + | > loader_time: 0.0104 (0.011726680095625592) + + + --> TIME: 2024-10-20 03:35:49 -- STEP: 44700/69059 -- GLOBAL_STEP: 44700 + | > loss_text_ce: 0.035391662269830704 (0.041649489579325905) + | > loss_mel_ce: 2.9752309322357178 (3.4116535854979455) + | > loss: 0.7526556253433228 (0.8633257687425197) + | > current_lr: 5e-06 + | > step_time: 0.4338 (0.24520051161981535) + | > loader_time: 0.0138 (0.011726163111010362) + + + --> TIME: 2024-10-20 03:36:06 -- STEP: 44750/69059 -- GLOBAL_STEP: 44750 + | > loss_text_ce: 0.03366200253367424 (0.041640983801896465) + | > loss_mel_ce: 3.2807884216308594 (3.411404846766788) + | > loss: 0.8286126255989075 (0.8632614576150595) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.24518919523867816) + | > loader_time: 0.0112 (0.011725986827019181) + + + --> TIME: 2024-10-20 03:36:26 -- STEP: 44800/69059 -- GLOBAL_STEP: 44800 + | > loss_text_ce: 0.031533077359199524 (0.04163244689710749) + | > loss_mel_ce: 3.4129934310913086 (3.4111472931823363) + | > loss: 0.8611316084861755 (0.8631949349918545) + | > current_lr: 5e-06 + | > step_time: 0.2752 (0.24520403411771569) + | > loader_time: 0.0123 (0.011726138080869376) + + + --> TIME: 2024-10-20 03:36:46 -- STEP: 44850/69059 -- GLOBAL_STEP: 44850 + | > loss_text_ce: 0.032911188900470734 (0.04162378923093207) + | > loss_mel_ce: 3.6298763751983643 (3.410938843399632) + | > loss: 0.915696918964386 (0.8631406581255647) + | > current_lr: 5e-06 + | > step_time: 0.494 (0.24522283593946476) + | > loader_time: 0.0152 (0.011726129041733393) + + + --> TIME: 2024-10-20 03:37:04 -- STEP: 44900/69059 -- GLOBAL_STEP: 44900 + | > loss_text_ce: 0.033311132341623306 (0.04161541126460974) + | > loss_mel_ce: 3.3480913639068604 (3.4106834852403223) + | > loss: 0.8453506231307983 (0.8630747240954368) + | > current_lr: 5e-06 + | > step_time: 0.1901 (0.2452204760358169) + | > loader_time: 0.0111 (0.01172601071657208) + + + --> TIME: 2024-10-20 03:37:22 -- STEP: 44950/69059 -- GLOBAL_STEP: 44950 + | > loss_text_ce: 0.03160226345062256 (0.04160642388785354) + | > loss_mel_ce: 2.9550111293792725 (3.4104560580693644) + | > loss: 0.7466533184051514 (0.8630156204591746) + | > current_lr: 5e-06 + | > step_time: 0.172 (0.24521247991597958) + | > loader_time: 0.0104 (0.011725669108720697) + + + --> TIME: 2024-10-20 03:37:41 -- STEP: 45000/69059 -- GLOBAL_STEP: 45000 + | > loss_text_ce: 0.0342291034758091 (0.041597839752956674) + | > loss_mel_ce: 3.321380138397217 (3.410204526101199) + | > loss: 0.8389022946357727 (0.8629505914343738) + | > current_lr: 5e-06 + | > step_time: 0.5011 (0.24520847996075948) + | > loader_time: 0.0152 (0.011725148778491524) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_45000.pth + + --> TIME: 2024-10-20 03:38:14 -- STEP: 45050/69059 -- GLOBAL_STEP: 45050 + | > loss_text_ce: 0.03273371234536171 (0.0415888622309697) + | > loss_mel_ce: 3.4647364616394043 (3.4099802805289645) + | > loss: 0.8743675351142883 (0.8628922856608199) + | > current_lr: 5e-06 + | > step_time: 0.4071 (0.24520971351140877) + | > loader_time: 0.0137 (0.011724849721038488) + + + --> TIME: 2024-10-20 03:38:33 -- STEP: 45100/69059 -- GLOBAL_STEP: 45100 + | > loss_text_ce: 0.029934171587228775 (0.04158065528354809) + | > loss_mel_ce: 2.794306516647339 (3.4097586705626317) + | > loss: 0.7060601711273193 (0.8628348314339209) + | > current_lr: 5e-06 + | > step_time: 0.1518 (0.2452269465801721) + | > loader_time: 0.0101 (0.011724813486678613) + + + --> TIME: 2024-10-20 03:38:53 -- STEP: 45150/69059 -- GLOBAL_STEP: 45150 + | > loss_text_ce: 0.03571509197354317 (0.04157218919615161) + | > loss_mel_ce: 2.9795031547546387 (3.409588898170807) + | > loss: 0.7538045644760132 (0.8627902718160645) + | > current_lr: 5e-06 + | > step_time: 0.1764 (0.24524013447471102) + | > loader_time: 0.0103 (0.011724715195884975) + + + --> TIME: 2024-10-20 03:39:11 -- STEP: 45200/69059 -- GLOBAL_STEP: 45200 + | > loss_text_ce: 0.03426855802536011 (0.04156379421942082) + | > loss_mel_ce: 3.6711044311523438 (3.40936288921177) + | > loss: 0.9263432621955872 (0.8627316708348504) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.24523749010225313) + | > loader_time: 0.0113 (0.011724534609676438) + + + --> TIME: 2024-10-20 03:39:28 -- STEP: 45250/69059 -- GLOBAL_STEP: 45250 + | > loss_text_ce: 0.03317517042160034 (0.04155514130889495) + | > loss_mel_ce: 3.232689619064331 (3.4091083102515856) + | > loss: 0.816466212272644 (0.8626658628658704) + | > current_lr: 5e-06 + | > step_time: 0.1939 (0.24521072125830046) + | > loader_time: 0.0106 (0.011724002147906368) + + + --> TIME: 2024-10-20 03:39:47 -- STEP: 45300/69059 -- GLOBAL_STEP: 45300 + | > loss_text_ce: 0.03147082030773163 (0.0415468960803482) + | > loss_mel_ce: 2.8158388137817383 (3.4088661376792935) + | > loss: 0.7118273973464966 (0.8626032584142485) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.24522198251804242) + | > loader_time: 0.0105 (0.011723910835931344) + + + --> TIME: 2024-10-20 03:40:06 -- STEP: 45350/69059 -- GLOBAL_STEP: 45350 + | > loss_text_ce: 0.03634792938828468 (0.04153859165107909) + | > loss_mel_ce: 3.3311705589294434 (3.4086042558160945) + | > loss: 0.8418796062469482 (0.8625357118423097) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.2452210184179304) + | > loader_time: 0.0109 (0.01172368786169384) + + + --> TIME: 2024-10-20 03:40:24 -- STEP: 45400/69059 -- GLOBAL_STEP: 45400 + | > loss_text_ce: 0.03405598923563957 (0.0415299264816685) + | > loss_mel_ce: 3.242006778717041 (3.4083871529700804) + | > loss: 0.8190156817436218 (0.8624792698393319) + | > current_lr: 5e-06 + | > step_time: 0.2493 (0.24522174081613313) + | > loader_time: 0.0138 (0.011723348133364413) + + + --> TIME: 2024-10-20 03:40:42 -- STEP: 45450/69059 -- GLOBAL_STEP: 45450 + | > loss_text_ce: 0.03786318749189377 (0.04152103628434508) + | > loss_mel_ce: 2.847862720489502 (3.4081423125723216) + | > loss: 0.7214314937591553 (0.8624158371924314) + | > current_lr: 5e-06 + | > step_time: 0.1773 (0.24520338357478477) + | > loader_time: 0.0109 (0.01172289529911622) + + + --> TIME: 2024-10-20 03:40:59 -- STEP: 45500/69059 -- GLOBAL_STEP: 45500 + | > loss_text_ce: 0.03282492235302925 (0.04151223830940088) + | > loss_mel_ce: 3.046403408050537 (3.4078956594938568) + | > loss: 0.7698071002960205 (0.862351974433596) + | > current_lr: 5e-06 + | > step_time: 0.1739 (0.24518625203331748) + | > loader_time: 0.0104 (0.011722435945992916) + + + --> TIME: 2024-10-20 03:41:17 -- STEP: 45550/69059 -- GLOBAL_STEP: 45550 + | > loss_text_ce: 0.03716416656970978 (0.04150425927069408) + | > loss_mel_ce: 3.266585350036621 (3.4076475197240175) + | > loss: 0.8259373903274536 (0.8622879447306299) + | > current_lr: 5e-06 + | > step_time: 0.5741 (0.24518112070604947) + | > loader_time: 0.0199 (0.011722556636571598) + + + --> TIME: 2024-10-20 03:41:35 -- STEP: 45600/69059 -- GLOBAL_STEP: 45600 + | > loss_text_ce: 0.035398051142692566 (0.04149600179771955) + | > loss_mel_ce: 3.36149263381958 (3.407413117707802) + | > loss: 0.8492226600646973 (0.8622272798575863) + | > current_lr: 5e-06 + | > step_time: 0.2462 (0.24517114296294096) + | > loader_time: 0.0115 (0.01172210814660053) + + + --> TIME: 2024-10-20 03:41:54 -- STEP: 45650/69059 -- GLOBAL_STEP: 45650 + | > loss_text_ce: 0.032913025468587875 (0.041487711921358215) + | > loss_mel_ce: 3.1302740573883057 (3.407158842797067) + | > loss: 0.7907967567443848 (0.8621616386596902) + | > current_lr: 5e-06 + | > step_time: 0.3027 (0.2451705940660224) + | > loader_time: 0.0111 (0.011721813566452285) + + + --> TIME: 2024-10-20 03:42:14 -- STEP: 45700/69059 -- GLOBAL_STEP: 45700 + | > loss_text_ce: 0.03632528707385063 (0.04147917398188365) + | > loss_mel_ce: 3.3779165744781494 (3.4068815988628103) + | > loss: 0.8535604476928711 (0.8620901931949475) + | > current_lr: 5e-06 + | > step_time: 0.3367 (0.24519149212138136) + | > loader_time: 0.0124 (0.011721719453840837) + + + --> TIME: 2024-10-20 03:42:30 -- STEP: 45750/69059 -- GLOBAL_STEP: 45750 + | > loss_text_ce: 0.03406882286071777 (0.04147067270362746) + | > loss_mel_ce: 3.0588338375091553 (3.406633102615004) + | > loss: 0.7732256650924683 (0.862025943814732) + | > current_lr: 5e-06 + | > step_time: 0.2341 (0.2451626592333851) + | > loader_time: 0.0116 (0.011720834263035483) + + + --> TIME: 2024-10-20 03:42:50 -- STEP: 45800/69059 -- GLOBAL_STEP: 45800 + | > loss_text_ce: 0.03260765224695206 (0.041462173327298776) + | > loss_mel_ce: 3.187386989593506 (3.406438619085776) + | > loss: 0.8049986362457275 (0.8619751980900774) + | > current_lr: 5e-06 + | > step_time: 0.1708 (0.24517729045522266) + | > loader_time: 0.0107 (0.01172079717748548) + + + --> TIME: 2024-10-20 03:43:08 -- STEP: 45850/69059 -- GLOBAL_STEP: 45850 + | > loss_text_ce: 0.031832292675971985 (0.041453640198409664) + | > loss_mel_ce: 2.963970184326172 (3.4062418439265887) + | > loss: 0.7489506006240845 (0.8619238710234453) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.24517390944315537) + | > loader_time: 0.0113 (0.01172054505530202) + + + --> TIME: 2024-10-20 03:43:27 -- STEP: 45900/69059 -- GLOBAL_STEP: 45900 + | > loss_text_ce: 0.033769045025110245 (0.04144581981620054) + | > loss_mel_ce: 3.052983522415161 (3.406001547920399) + | > loss: 0.7716881632804871 (0.8618618419331413) + | > current_lr: 5e-06 + | > step_time: 0.2058 (0.2451751909588417) + | > loader_time: 0.0108 (0.011720138401247552) + + + --> TIME: 2024-10-20 03:43:44 -- STEP: 45950/69059 -- GLOBAL_STEP: 45950 + | > loss_text_ce: 0.03412950038909912 (0.04143746854605609) + | > loss_mel_ce: 3.189939022064209 (3.4057881812509505) + | > loss: 0.8060171604156494 (0.8618064124488217) + | > current_lr: 5e-06 + | > step_time: 0.1962 (0.24515784570779064) + | > loader_time: 0.0104 (0.01171980053089127) + + + --> TIME: 2024-10-20 03:44:02 -- STEP: 46000/69059 -- GLOBAL_STEP: 46000 + | > loss_text_ce: 0.03134572505950928 (0.04142915419914262) + | > loss_mel_ce: 3.0196547508239746 (3.4055287816057844) + | > loss: 0.7627501487731934 (0.8617394839525232) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.24514546069373255) + | > loader_time: 0.0103 (0.011719419826631937) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_46000.pth + + --> TIME: 2024-10-20 03:44:35 -- STEP: 46050/69059 -- GLOBAL_STEP: 46050 + | > loss_text_ce: 0.03302028030157089 (0.0414211183483429) + | > loss_mel_ce: 3.675208330154419 (3.4053191521061428) + | > loss: 0.927057147026062 (0.8616850676132733) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.24513997467281246) + | > loader_time: 0.0106 (0.011719165008826547) + + + --> TIME: 2024-10-20 03:44:54 -- STEP: 46100/69059 -- GLOBAL_STEP: 46100 + | > loss_text_ce: 0.035183485597372055 (0.041412862419814886) + | > loss_mel_ce: 3.4896092414855957 (3.405068194633432) + | > loss: 0.8811981678009033 (0.8616202642628024) + | > current_lr: 5e-06 + | > step_time: 0.3626 (0.24513885469602142) + | > loader_time: 0.0126 (0.011718971532751294) + + + --> TIME: 2024-10-20 03:45:13 -- STEP: 46150/69059 -- GLOBAL_STEP: 46150 + | > loss_text_ce: 0.03091469407081604 (0.04140480767584325) + | > loss_mel_ce: 3.141467332839966 (3.4048232511373477) + | > loss: 0.7930955290794373 (0.8615570147027183) + | > current_lr: 5e-06 + | > step_time: 0.4546 (0.24514984526071104) + | > loader_time: 0.0136 (0.011719467110391015) + + + --> TIME: 2024-10-20 03:45:33 -- STEP: 46200/69059 -- GLOBAL_STEP: 46200 + | > loss_text_ce: 0.03076831065118313 (0.04139669530029113) + | > loss_mel_ce: 3.434495449066162 (3.404606028181115) + | > loss: 0.8663159608840942 (0.8615006808723732) + | > current_lr: 5e-06 + | > step_time: 0.3779 (0.24516863107165218) + | > loader_time: 0.0129 (0.011719431990668862) + + + --> TIME: 2024-10-20 03:45:52 -- STEP: 46250/69059 -- GLOBAL_STEP: 46250 + | > loss_text_ce: 0.034555498510599136 (0.041389083162432894) + | > loss_mel_ce: 2.9307618141174316 (3.404344282165715) + | > loss: 0.7413293123245239 (0.8614333413343181) + | > current_lr: 5e-06 + | > step_time: 0.2779 (0.24518512673764614) + | > loader_time: 0.0102 (0.01171949904673806) + + + --> TIME: 2024-10-20 03:46:12 -- STEP: 46300/69059 -- GLOBAL_STEP: 46300 + | > loss_text_ce: 0.030039643868803978 (0.04138105977443245) + | > loss_mel_ce: 3.341576337814331 (3.404129313830404) + | > loss: 0.8429039716720581 (0.8613775934011848) + | > current_lr: 5e-06 + | > step_time: 0.173 (0.24520048365994612) + | > loader_time: 0.0111 (0.011719496893831232) + + + --> TIME: 2024-10-20 03:46:29 -- STEP: 46350/69059 -- GLOBAL_STEP: 46350 + | > loss_text_ce: 0.036776378750801086 (0.04137246328300492) + | > loss_mel_ce: 3.2511391639709473 (3.403924307452803) + | > loss: 0.8219788670539856 (0.861324192680331) + | > current_lr: 5e-06 + | > step_time: 0.2053 (0.2451861677612173) + | > loader_time: 0.0114 (0.01171937524822278) + + + --> TIME: 2024-10-20 03:46:48 -- STEP: 46400/69059 -- GLOBAL_STEP: 46400 + | > loss_text_ce: 0.0317998006939888 (0.04136427399646113) + | > loss_mel_ce: 2.727142810821533 (3.4036559723625737) + | > loss: 0.6897356510162354 (0.8612550615891824) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.24519424502191872) + | > loader_time: 0.012 (0.011719354956314458) + + + --> TIME: 2024-10-20 03:47:08 -- STEP: 46450/69059 -- GLOBAL_STEP: 46450 + | > loss_text_ce: 0.03195206820964813 (0.041356670576898034) + | > loss_mel_ce: 3.527600049972534 (3.403384896869661) + | > loss: 0.8898880481719971 (0.8611853918577559) + | > current_lr: 5e-06 + | > step_time: 0.2664 (0.24521876753467275) + | > loader_time: 0.0118 (0.011719399391642403) + + + --> TIME: 2024-10-20 03:47:27 -- STEP: 46500/69059 -- GLOBAL_STEP: 46500 + | > loss_text_ce: 0.03330406919121742 (0.04134843226626301) + | > loss_mel_ce: 3.1072583198547363 (3.40310877797935) + | > loss: 0.7851405739784241 (0.8611143025582846) + | > current_lr: 5e-06 + | > step_time: 0.1707 (0.2452152939201683) + | > loader_time: 0.0105 (0.011721650549160512) + + + --> TIME: 2024-10-20 03:47:46 -- STEP: 46550/69059 -- GLOBAL_STEP: 46550 + | > loss_text_ce: 0.03455745801329613 (0.04134012420806741) + | > loss_mel_ce: 3.464200258255005 (3.4029024337518425) + | > loss: 0.8746894001960754 (0.8610606394905034) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.24523082149272063) + | > loader_time: 0.0101 (0.011721530150137938) + + + --> TIME: 2024-10-20 03:48:06 -- STEP: 46600/69059 -- GLOBAL_STEP: 46600 + | > loss_text_ce: 0.034875817596912384 (0.04133228858621024) + | > loss_mel_ce: 3.0171875953674316 (3.402667906181988) + | > loss: 0.763015866279602 (0.8610000486946934) + | > current_lr: 5e-06 + | > step_time: 0.1641 (0.2452418720108245) + | > loader_time: 0.0107 (0.011721278142519751) + + + --> TIME: 2024-10-20 03:48:23 -- STEP: 46650/69059 -- GLOBAL_STEP: 46650 + | > loss_text_ce: 0.03466148301959038 (0.04132381522003805) + | > loss_mel_ce: 3.0804548263549805 (3.40243343966565) + | > loss: 0.7787790894508362 (0.8609393137252609) + | > current_lr: 5e-06 + | > step_time: 0.2317 (0.24522527591464094) + | > loader_time: 0.0118 (0.011720817188741286) + + + --> TIME: 2024-10-20 03:48:41 -- STEP: 46700/69059 -- GLOBAL_STEP: 46700 + | > loss_text_ce: 0.03296393156051636 (0.04131571434761242) + | > loss_mel_ce: 3.076266288757324 (3.4021611718038742) + | > loss: 0.7773075699806213 (0.8608692215405398) + | > current_lr: 5e-06 + | > step_time: 0.1972 (0.24521444405937604) + | > loader_time: 0.0112 (0.011720248095739043) + + + --> TIME: 2024-10-20 03:48:58 -- STEP: 46750/69059 -- GLOBAL_STEP: 46750 + | > loss_text_ce: 0.03610753268003464 (0.04130707296282793) + | > loss_mel_ce: 3.212556838989258 (3.4018844637284116) + | > loss: 0.8121660947799683 (0.8607978841799475) + | > current_lr: 5e-06 + | > step_time: 0.4419 (0.24519307420597994) + | > loader_time: 0.0131 (0.011719615574189024) + + + --> TIME: 2024-10-20 03:49:16 -- STEP: 46800/69059 -- GLOBAL_STEP: 46800 + | > loss_text_ce: 0.03191694617271423 (0.041299005031593054) + | > loss_mel_ce: 3.1971752643585205 (3.4016462336493167) + | > loss: 0.8072730302810669 (0.8607363096745615) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.24518403301381658) + | > loader_time: 0.0101 (0.011719059042441514) + + + --> TIME: 2024-10-20 03:49:37 -- STEP: 46850/69059 -- GLOBAL_STEP: 46850 + | > loss_text_ce: 0.03276848793029785 (0.041291310710865656) + | > loss_mel_ce: 3.1038458347320557 (3.4014098032866693) + | > loss: 0.7841535806655884 (0.8606752785051145) + | > current_lr: 5e-06 + | > step_time: 0.2343 (0.2452081449767187) + | > loader_time: 0.0126 (0.011718993268374294) + + + --> TIME: 2024-10-20 03:49:54 -- STEP: 46900/69059 -- GLOBAL_STEP: 46900 + | > loss_text_ce: 0.03251724690198898 (0.04128310254999319) + | > loss_mel_ce: 3.3709771633148193 (3.4011266862303784) + | > loss: 0.850873589515686 (0.8606024472011949) + | > current_lr: 5e-06 + | > step_time: 0.1814 (0.24518717922889857) + | > loader_time: 0.0107 (0.011718591779533964) + + + --> TIME: 2024-10-20 03:50:15 -- STEP: 46950/69059 -- GLOBAL_STEP: 46950 + | > loss_text_ce: 0.03286237269639969 (0.04127550955119867) + | > loss_mel_ce: 3.240765333175659 (3.400885572235406) + | > loss: 0.8184069395065308 (0.8605402704536987) + | > current_lr: 5e-06 + | > step_time: 0.2784 (0.2452288904687721) + | > loader_time: 0.0133 (0.011718577449948872) + + + --> TIME: 2024-10-20 03:50:33 -- STEP: 47000/69059 -- GLOBAL_STEP: 47000 + | > loss_text_ce: 0.03492625430226326 (0.04126743953960316) + | > loss_mel_ce: 3.1050779819488525 (3.4006048652415393) + | > loss: 0.7850010395050049 (0.8604680761973917) + | > current_lr: 5e-06 + | > step_time: 0.3618 (0.24521935996603458) + | > loader_time: 0.0155 (0.01171814659808542) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_47000.pth + + --> TIME: 2024-10-20 03:51:07 -- STEP: 47050/69059 -- GLOBAL_STEP: 47050 + | > loss_text_ce: 0.03376690670847893 (0.041259229694325314) + | > loss_mel_ce: 2.981984853744507 (3.400339619340584) + | > loss: 0.7539379596710205 (0.8603997122646021) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.24520933539910472) + | > loader_time: 0.0115 (0.011717961206953002) + + + --> TIME: 2024-10-20 03:51:25 -- STEP: 47100/69059 -- GLOBAL_STEP: 47100 + | > loss_text_ce: 0.03616872429847717 (0.041250982726518226) + | > loss_mel_ce: 3.407513380050659 (3.400095895511058) + | > loss: 0.8609205484390259 (0.8603367195653318) + | > current_lr: 5e-06 + | > step_time: 0.2272 (0.245200218992375) + | > loader_time: 0.0113 (0.011717915033838528) + + + --> TIME: 2024-10-20 03:51:42 -- STEP: 47150/69059 -- GLOBAL_STEP: 47150 + | > loss_text_ce: 0.03246403485536575 (0.041242719057824395) + | > loss_mel_ce: 2.7282352447509766 (3.399801198488057) + | > loss: 0.6901748180389404 (0.8602609793909796) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24517673974325396) + | > loader_time: 0.0108 (0.01171761403533836) + + + --> TIME: 2024-10-20 03:52:00 -- STEP: 47200/69059 -- GLOBAL_STEP: 47200 + | > loss_text_ce: 0.03606514632701874 (0.04123486431792988) + | > loss_mel_ce: 3.3318610191345215 (3.399585344316577) + | > loss: 0.8419815301895142 (0.8602050521654102) + | > current_lr: 5e-06 + | > step_time: 0.1815 (0.24517479273222262) + | > loader_time: 0.0105 (0.011717659356230372) + + + --> TIME: 2024-10-20 03:52:19 -- STEP: 47250/69059 -- GLOBAL_STEP: 47250 + | > loss_text_ce: 0.033114541321992874 (0.041226612752076265) + | > loss_mel_ce: 3.0276665687561035 (3.3993262580185157) + | > loss: 0.7651952505111694 (0.8601382176951766) + | > current_lr: 5e-06 + | > step_time: 0.2168 (0.2451870726832637) + | > loader_time: 0.0117 (0.01171771443079386) + + + --> TIME: 2024-10-20 03:52:39 -- STEP: 47300/69059 -- GLOBAL_STEP: 47300 + | > loss_text_ce: 0.03277383744716644 (0.04121860665771228) + | > loss_mel_ce: 3.258826494216919 (3.3990811320238152) + | > loss: 0.8229000568389893 (0.8600749346708404) + | > current_lr: 5e-06 + | > step_time: 0.1732 (0.2451970144759777) + | > loader_time: 0.0106 (0.011717859811561027) + + + --> TIME: 2024-10-20 03:52:57 -- STEP: 47350/69059 -- GLOBAL_STEP: 47350 + | > loss_text_ce: 0.032144371420145035 (0.04121041320008589) + | > loss_mel_ce: 3.2829666137695312 (3.3988370779776464) + | > loss: 0.8287777304649353 (0.8600118727882922) + | > current_lr: 5e-06 + | > step_time: 0.2049 (0.24520143686151555) + | > loader_time: 0.0113 (0.01171774167319661) + + + --> TIME: 2024-10-20 03:53:15 -- STEP: 47400/69059 -- GLOBAL_STEP: 47400 + | > loss_text_ce: 0.03186222165822983 (0.04120274710264551) + | > loss_mel_ce: 3.0915913581848145 (3.3985924451109546) + | > loss: 0.7808634042739868 (0.8599487980496038) + | > current_lr: 5e-06 + | > step_time: 0.1639 (0.24518670414067523) + | > loader_time: 0.0108 (0.011717368756668452) + + + --> TIME: 2024-10-20 03:53:33 -- STEP: 47450/69059 -- GLOBAL_STEP: 47450 + | > loss_text_ce: 0.03166109323501587 (0.04119460815789459) + | > loss_mel_ce: 3.0508930683135986 (3.398347747253022) + | > loss: 0.7706385254859924 (0.8598855888504383) + | > current_lr: 5e-06 + | > step_time: 0.2236 (0.24517833625805516) + | > loader_time: 0.0116 (0.011717450063522523) + + + --> TIME: 2024-10-20 03:53:52 -- STEP: 47500/69059 -- GLOBAL_STEP: 47500 + | > loss_text_ce: 0.03344668075442314 (0.04118737643321822) + | > loss_mel_ce: 3.5282809734344482 (3.3981067191826484) + | > loss: 0.8904319405555725 (0.8598235239041488) + | > current_lr: 5e-06 + | > step_time: 0.318 (0.24518374131353277) + | > loader_time: 0.0136 (0.011717377281188941) + + + --> TIME: 2024-10-20 03:54:13 -- STEP: 47550/69059 -- GLOBAL_STEP: 47550 + | > loss_text_ce: 0.03077084943652153 (0.04117994639728666) + | > loss_mel_ce: 3.0213260650634766 (3.3978992830892034) + | > loss: 0.7630242109298706 (0.8597698073657914) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.24521805105399885) + | > loader_time: 0.0139 (0.011717821476211284) + + + --> TIME: 2024-10-20 03:54:31 -- STEP: 47600/69059 -- GLOBAL_STEP: 47600 + | > loss_text_ce: 0.03231959044933319 (0.04117174235074685) + | > loss_mel_ce: 3.127129077911377 (3.397667404488337) + | > loss: 0.7898621559143066 (0.859709786703858) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.24520538006009174) + | > loader_time: 0.0109 (0.011717863107929686) + + + --> TIME: 2024-10-20 03:54:49 -- STEP: 47650/69059 -- GLOBAL_STEP: 47650 + | > loss_text_ce: 0.032562006264925 (0.041163894655055364) + | > loss_mel_ce: 3.1745972633361816 (3.3974597393828265) + | > loss: 0.8017898201942444 (0.8596559085025376) + | > current_lr: 5e-06 + | > step_time: 0.2233 (0.24520871410089676) + | > loader_time: 0.0109 (0.011717630883704179) + + + --> TIME: 2024-10-20 03:55:07 -- STEP: 47700/69059 -- GLOBAL_STEP: 47700 + | > loss_text_ce: 0.03416025638580322 (0.041155818363053646) + | > loss_mel_ce: 3.3217594623565674 (3.397234743281212) + | > loss: 0.838979959487915 (0.8595976404099104) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.24519316041494565) + | > loader_time: 0.0108 (0.011717196315589417) + + + --> TIME: 2024-10-20 03:55:27 -- STEP: 47750/69059 -- GLOBAL_STEP: 47750 + | > loss_text_ce: 0.03426596522331238 (0.041148248648962805) + | > loss_mel_ce: 3.2193491458892822 (3.396993079305301) + | > loss: 0.8134037852287292 (0.859535331984466) + | > current_lr: 5e-06 + | > step_time: 0.2838 (0.24522334541700272) + | > loader_time: 0.0123 (0.0117175036375435) + + + --> TIME: 2024-10-20 03:55:45 -- STEP: 47800/69059 -- GLOBAL_STEP: 47800 + | > loss_text_ce: 0.031505703926086426 (0.04114059183360741) + | > loss_mel_ce: 3.277975082397461 (3.3967418324697785) + | > loss: 0.8273701667785645 (0.8594706060721794) + | > current_lr: 5e-06 + | > step_time: 0.1907 (0.24520435120770123) + | > loader_time: 0.0107 (0.011717206359408367) + + + --> TIME: 2024-10-20 03:56:02 -- STEP: 47850/69059 -- GLOBAL_STEP: 47850 + | > loss_text_ce: 0.03515063598752022 (0.0411330747209258) + | > loss_mel_ce: 3.467897891998291 (3.3964978887444195) + | > loss: 0.8757621049880981 (0.8594077408612718) + | > current_lr: 5e-06 + | > step_time: 0.2859 (0.24518996423563863) + | > loader_time: 0.0122 (0.011716805784928987) + + + --> TIME: 2024-10-20 03:56:21 -- STEP: 47900/69059 -- GLOBAL_STEP: 47900 + | > loss_text_ce: 0.033371079713106155 (0.04112569310696493) + | > loss_mel_ce: 2.927650213241577 (3.3962163210711367) + | > loss: 0.7402552962303162 (0.8593355035371232) + | > current_lr: 5e-06 + | > step_time: 0.1544 (0.24520408590551707) + | > loader_time: 0.0135 (0.011717151311344768) + + + --> TIME: 2024-10-20 03:56:40 -- STEP: 47950/69059 -- GLOBAL_STEP: 47950 + | > loss_text_ce: 0.035616155713796616 (0.04111812781460908) + | > loss_mel_ce: 2.9751787185668945 (3.3959436155384664) + | > loss: 0.7526987195014954 (0.8592654358284078) + | > current_lr: 5e-06 + | > step_time: 0.1871 (0.2452038778726699) + | > loader_time: 0.0107 (0.011717124268707793) + + + --> TIME: 2024-10-20 03:57:00 -- STEP: 48000/69059 -- GLOBAL_STEP: 48000 + | > loss_text_ce: 0.034405481070280075 (0.04111047271712781) + | > loss_mel_ce: 2.844733953475952 (3.3957906675487575) + | > loss: 0.7197848558425903 (0.8592252850619464) + | > current_lr: 5e-06 + | > step_time: 0.2044 (0.24522872470815976) + | > loader_time: 0.0109 (0.01171713325877982) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_48000.pth + + --> TIME: 2024-10-20 03:57:33 -- STEP: 48050/69059 -- GLOBAL_STEP: 48050 + | > loss_text_ce: 0.03271989896893501 (0.04110244743863181) + | > loss_mel_ce: 2.998337507247925 (3.3955626641103622) + | > loss: 0.7577643394470215 (0.8591662778827088) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.2452288974559518) + | > loader_time: 0.0105 (0.011717293939778982) + + + --> TIME: 2024-10-20 03:57:52 -- STEP: 48100/69059 -- GLOBAL_STEP: 48100 + | > loss_text_ce: 0.032276298850774765 (0.04109459979521549) + | > loss_mel_ce: 3.158888816833496 (3.395322069199796) + | > loss: 0.7977913022041321 (0.8591041672428045) + | > current_lr: 5e-06 + | > step_time: 0.1824 (0.2452233952990193) + | > loader_time: 0.0108 (0.011717211645011324) + + + --> TIME: 2024-10-20 03:58:08 -- STEP: 48150/69059 -- GLOBAL_STEP: 48150 + | > loss_text_ce: 0.03226214647293091 (0.04108636747906365) + | > loss_mel_ce: 2.768409252166748 (3.3950139151233323) + | > loss: 0.7001678347587585 (0.8590250706425099) + | > current_lr: 5e-06 + | > step_time: 0.1947 (0.2451965337676051) + | > loader_time: 0.0111 (0.011716635353468636) + + + --> TIME: 2024-10-20 03:58:26 -- STEP: 48200/69059 -- GLOBAL_STEP: 48200 + | > loss_text_ce: 0.034589119255542755 (0.0410786387961955) + | > loss_mel_ce: 3.444087266921997 (3.394789715035804) + | > loss: 0.8696690797805786 (0.8589670884460838) + | > current_lr: 5e-06 + | > step_time: 0.5034 (0.24518703994414617) + | > loader_time: 0.0165 (0.011716479315302656) + + + --> TIME: 2024-10-20 03:58:47 -- STEP: 48250/69059 -- GLOBAL_STEP: 48250 + | > loss_text_ce: 0.03391096368432045 (0.04107120753122061) + | > loss_mel_ce: 2.9220986366271973 (3.3945884583280312) + | > loss: 0.7390024065971375 (0.8589149164597614) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.24521911145244854) + | > loader_time: 0.0115 (0.011716946804461678) + + + --> TIME: 2024-10-20 03:59:04 -- STEP: 48300/69059 -- GLOBAL_STEP: 48300 + | > loss_text_ce: 0.034260284155607224 (0.041063295241799504) + | > loss_mel_ce: 3.0881896018981934 (3.3943300977207183) + | > loss: 0.7806124687194824 (0.8588483482377138) + | > current_lr: 5e-06 + | > step_time: 0.182 (0.2451919308212233) + | > loader_time: 0.0121 (0.011716503405916495) + + + --> TIME: 2024-10-20 03:59:23 -- STEP: 48350/69059 -- GLOBAL_STEP: 48350 + | > loss_text_ce: 0.03427201509475708 (0.04105589367853729) + | > loss_mel_ce: 3.339323043823242 (3.3941268894171706) + | > loss: 0.8433987498283386 (0.8587956957683947) + | > current_lr: 5e-06 + | > step_time: 0.2862 (0.24520523944093375) + | > loader_time: 0.0117 (0.011716391648230386) + + + --> TIME: 2024-10-20 03:59:41 -- STEP: 48400/69059 -- GLOBAL_STEP: 48400 + | > loss_text_ce: 0.03031454049050808 (0.04104838826081632) + | > loss_mel_ce: 3.5094940662384033 (3.393903928231584) + | > loss: 0.8849521279335022 (0.8587380791215373) + | > current_lr: 5e-06 + | > step_time: 0.1818 (0.2451992091561152) + | > loader_time: 0.0111 (0.011716585715940152) + + + --> TIME: 2024-10-20 04:00:00 -- STEP: 48450/69059 -- GLOBAL_STEP: 48450 + | > loss_text_ce: 0.031875770539045334 (0.04104050512403436) + | > loss_mel_ce: 3.239543914794922 (3.3936883757997536) + | > loss: 0.8178549408912659 (0.8586822202281941) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.2452023726523353) + | > loader_time: 0.011 (0.011716326619080323) + + + --> TIME: 2024-10-20 04:00:19 -- STEP: 48500/69059 -- GLOBAL_STEP: 48500 + | > loss_text_ce: 0.03494719788432121 (0.04103307939896816) + | > loss_mel_ce: 3.836886405944824 (3.3934587235548883) + | > loss: 0.967958390712738 (0.8586229507320944) + | > current_lr: 5e-06 + | > step_time: 0.4646 (0.24521045260085272) + | > loader_time: 0.0155 (0.011716339239140124) + + + --> TIME: 2024-10-20 04:00:38 -- STEP: 48550/69059 -- GLOBAL_STEP: 48550 + | > loss_text_ce: 0.03194419667124748 (0.04102545917809069) + | > loss_mel_ce: 2.89485764503479 (3.3932619250078346) + | > loss: 0.7317004799842834 (0.8585718460407129) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.24521198070380765) + | > loader_time: 0.0104 (0.011716293684608758) + + + --> TIME: 2024-10-20 04:00:56 -- STEP: 48600/69059 -- GLOBAL_STEP: 48600 + | > loss_text_ce: 0.0319693498313427 (0.04101772954668251) + | > loss_mel_ce: 2.963743209838867 (3.3930298832904966) + | > loss: 0.7489281296730042 (0.8585119032111687) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.24520611134085635) + | > loader_time: 0.0109 (0.011716027097937478) + + + --> TIME: 2024-10-20 04:01:16 -- STEP: 48650/69059 -- GLOBAL_STEP: 48650 + | > loss_text_ce: 0.03390979394316673 (0.041010816207103284) + | > loss_mel_ce: 3.0026705265045166 (3.39284621425423) + | > loss: 0.7591450810432434 (0.858464257614219) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.24522181393798798) + | > loader_time: 0.0106 (0.011716041834479345) + + + --> TIME: 2024-10-20 04:01:34 -- STEP: 48700/69059 -- GLOBAL_STEP: 48700 + | > loss_text_ce: 0.03199175372719765 (0.04100273760161791) + | > loss_mel_ce: 2.9802427291870117 (3.39261842342371) + | > loss: 0.7530586123466492 (0.8584052902570259) + | > current_lr: 5e-06 + | > step_time: 0.1985 (0.24521224906067585) + | > loader_time: 0.0106 (0.011715491767781448) + + + --> TIME: 2024-10-20 04:01:51 -- STEP: 48750/69059 -- GLOBAL_STEP: 48750 + | > loss_text_ce: 0.03163208067417145 (0.04099504833290152) + | > loss_mel_ce: 3.08766508102417 (3.392358778630752) + | > loss: 0.7798243165016174 (0.8583384567419697) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.24519801761920634) + | > loader_time: 0.0112 (0.011715006686479594) + + + --> TIME: 2024-10-20 04:02:09 -- STEP: 48800/69059 -- GLOBAL_STEP: 48800 + | > loss_text_ce: 0.03309309110045433 (0.04098751897083871) + | > loss_mel_ce: 3.04646372795105 (3.392156796069401) + | > loss: 0.7698891758918762 (0.858286078757928) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.24518893830600333) + | > loader_time: 0.0113 (0.0117147229538589) + + + --> TIME: 2024-10-20 04:02:27 -- STEP: 48850/69059 -- GLOBAL_STEP: 48850 + | > loss_text_ce: 0.029533809050917625 (0.04098022800541305) + | > loss_mel_ce: 2.9422004222869873 (3.391891133270324) + | > loss: 0.7429335713386536 (0.8582178403215993) + | > current_lr: 5e-06 + | > step_time: 0.2312 (0.2451737562441167) + | > loader_time: 0.0112 (0.011714163275555653) + + + --> TIME: 2024-10-20 04:02:46 -- STEP: 48900/69059 -- GLOBAL_STEP: 48900 + | > loss_text_ce: 0.03439691662788391 (0.04097273769599827) + | > loss_mel_ce: 3.1554455757141113 (3.3917022529945116) + | > loss: 0.7974606156349182 (0.8581687476786133) + | > current_lr: 5e-06 + | > step_time: 0.3147 (0.24519344588730232) + | > loader_time: 0.0118 (0.01171395095579461) + + + --> TIME: 2024-10-20 04:03:05 -- STEP: 48950/69059 -- GLOBAL_STEP: 48950 + | > loss_text_ce: 0.03189784288406372 (0.04096537646827002) + | > loss_mel_ce: 3.098609447479248 (3.3914984496888945) + | > loss: 0.7826268076896667 (0.8581159565434158) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.24520109628145492) + | > loader_time: 0.0113 (0.01171375431493303) + + + --> TIME: 2024-10-20 04:03:25 -- STEP: 49000/69059 -- GLOBAL_STEP: 49000 + | > loss_text_ce: 0.03696785867214203 (0.040957925104250364) + | > loss_mel_ce: 3.3908822536468506 (3.3912775478411517) + | > loss: 0.8569625020027161 (0.8580588682391215) + | > current_lr: 5e-06 + | > step_time: 0.2673 (0.24522112057646928) + | > loader_time: 0.0112 (0.01171352647761907) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_49000.pth + + --> TIME: 2024-10-20 04:03:59 -- STEP: 49050/69059 -- GLOBAL_STEP: 49050 + | > loss_text_ce: 0.03522397577762604 (0.04095091251719009) + | > loss_mel_ce: 2.8451924324035645 (3.3910614085610433) + | > loss: 0.7201040983200073 (0.858003080275203) + | > current_lr: 5e-06 + | > step_time: 0.1677 (0.24523825280406303) + | > loader_time: 0.011 (0.01171397367627125) + + + --> TIME: 2024-10-20 04:04:18 -- STEP: 49100/69059 -- GLOBAL_STEP: 49100 + | > loss_text_ce: 0.03361111879348755 (0.04094334419046577) + | > loss_mel_ce: 3.2113592624664307 (3.3908232164139966) + | > loss: 0.8112425804138184 (0.8579416401590212) + | > current_lr: 5e-06 + | > step_time: 0.3224 (0.24525361713226845) + | > loader_time: 0.0128 (0.01171401093302334) + + + --> TIME: 2024-10-20 04:04:37 -- STEP: 49150/69059 -- GLOBAL_STEP: 49150 + | > loss_text_ce: 0.03244594857096672 (0.040936022383650135) + | > loss_mel_ce: 3.463953971862793 (3.390620959578775) + | > loss: 0.8740999698638916 (0.85788924549749) + | > current_lr: 5e-06 + | > step_time: 0.3237 (0.24525802787904943) + | > loader_time: 0.0122 (0.011714054696557457) + + + --> TIME: 2024-10-20 04:04:55 -- STEP: 49200/69059 -- GLOBAL_STEP: 49200 + | > loss_text_ce: 0.033315401524305344 (0.0409284050758483) + | > loss_mel_ce: 3.0969595909118652 (3.3903683410863388) + | > loss: 0.7825687527656555 (0.8578241865482281) + | > current_lr: 5e-06 + | > step_time: 0.1641 (0.24525505424999608) + | > loader_time: 0.0109 (0.011714000823051926) + + + --> TIME: 2024-10-20 04:05:12 -- STEP: 49250/69059 -- GLOBAL_STEP: 49250 + | > loss_text_ce: 0.03418795391917229 (0.04092079251872198) + | > loss_mel_ce: 3.202148914337158 (3.3901229120196246) + | > loss: 0.8090842366218567 (0.8577609261425627) + | > current_lr: 5e-06 + | > step_time: 0.1856 (0.24523398244925562) + | > loader_time: 0.0105 (0.011713734612247038) + + + --> TIME: 2024-10-20 04:05:31 -- STEP: 49300/69059 -- GLOBAL_STEP: 49300 + | > loss_text_ce: 0.029474694281816483 (0.04091321157280144) + | > loss_mel_ce: 3.290361166000366 (3.389899571609286) + | > loss: 0.829958975315094 (0.8577031958066919) + | > current_lr: 5e-06 + | > step_time: 0.1644 (0.24523998444752568) + | > loader_time: 0.0109 (0.011713728546854426) + + + --> TIME: 2024-10-20 04:05:50 -- STEP: 49350/69059 -- GLOBAL_STEP: 49350 + | > loss_text_ce: 0.029308447614312172 (0.04090552814844508) + | > loss_mel_ce: 3.2211174964904785 (3.389690511446455) + | > loss: 0.8126065135002136 (0.8576490099077417) + | > current_lr: 5e-06 + | > step_time: 0.188 (0.24524451018104437) + | > loader_time: 0.0141 (0.0117139100641948) + + + --> TIME: 2024-10-20 04:06:08 -- STEP: 49400/69059 -- GLOBAL_STEP: 49400 + | > loss_text_ce: 0.03236344829201698 (0.04089765021034464) + | > loss_mel_ce: 3.70615816116333 (3.3894433387474314) + | > loss: 0.9346303939819336 (0.8575852472482914) + | > current_lr: 5e-06 + | > step_time: 0.2422 (0.24522670982820302) + | > loader_time: 0.0126 (0.011713499931188706) + + + --> TIME: 2024-10-20 04:06:28 -- STEP: 49450/69059 -- GLOBAL_STEP: 49450 + | > loss_text_ce: 0.03226274996995926 (0.04089049233558662) + | > loss_mel_ce: 3.5667293071746826 (3.3892559051127784) + | > loss: 0.8997480273246765 (0.8575365993684907) + | > current_lr: 5e-06 + | > step_time: 0.2232 (0.2452503185783286) + | > loader_time: 0.0118 (0.011713536217190958) + + + --> TIME: 2024-10-20 04:06:47 -- STEP: 49500/69059 -- GLOBAL_STEP: 49500 + | > loss_text_ce: 0.03607206791639328 (0.04088334732453728) + | > loss_mel_ce: 3.1876063346862793 (3.3891002039620193) + | > loss: 0.8059195876121521 (0.8574958878310048) + | > current_lr: 5e-06 + | > step_time: 0.3146 (0.24526489109944816) + | > loader_time: 0.0122 (0.0117140543822086) + + + --> TIME: 2024-10-20 04:07:06 -- STEP: 49550/69059 -- GLOBAL_STEP: 49550 + | > loss_text_ce: 0.03488170728087425 (0.040876114967679913) + | > loss_mel_ce: 3.37294340133667 (3.3889283232154632) + | > loss: 0.8519562482833862 (0.8574511095481501) + | > current_lr: 5e-06 + | > step_time: 0.2014 (0.2452727524615922) + | > loader_time: 0.0112 (0.01171395864785016) + + + --> TIME: 2024-10-20 04:07:24 -- STEP: 49600/69059 -- GLOBAL_STEP: 49600 + | > loss_text_ce: 0.03544643893837929 (0.04086861661684109) + | > loss_mel_ce: 2.9518020153045654 (3.388674308845095) + | > loss: 0.746812105178833 (0.8573857313694982) + | > current_lr: 5e-06 + | > step_time: 0.1579 (0.24525797742509073) + | > loader_time: 0.0101 (0.011713451416261712) + + + --> TIME: 2024-10-20 04:07:43 -- STEP: 49650/69059 -- GLOBAL_STEP: 49650 + | > loss_text_ce: 0.0344095453619957 (0.04086119677012306) + | > loss_mel_ce: 3.2071633338928223 (3.388526892873318) + | > loss: 0.810393214225769 (0.8573470224150487) + | > current_lr: 5e-06 + | > step_time: 0.3671 (0.24527144913707014) + | > loader_time: 0.0134 (0.011713679535511374) + + + --> TIME: 2024-10-20 04:08:01 -- STEP: 49700/69059 -- GLOBAL_STEP: 49700 + | > loss_text_ce: 0.03246517479419708 (0.04085433267502874) + | > loss_mel_ce: 3.221754312515259 (3.388246262179037) + | > loss: 0.8135548830032349 (0.8572751487152925) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24526112268148534) + | > loader_time: 0.0105 (0.01171323277341046) + + + --> TIME: 2024-10-20 04:08:19 -- STEP: 49750/69059 -- GLOBAL_STEP: 49750 + | > loss_text_ce: 0.03683717921376228 (0.04084708332959226) + | > loss_mel_ce: 3.3233227729797363 (3.3880438753204554) + | > loss: 0.8400399684906006 (0.8572227396665515) + | > current_lr: 5e-06 + | > step_time: 0.2876 (0.24525995952759555) + | > loader_time: 0.0129 (0.01171279996843192) + + + --> TIME: 2024-10-20 04:08:37 -- STEP: 49800/69059 -- GLOBAL_STEP: 49800 + | > loss_text_ce: 0.0362030491232872 (0.04083908936498729) + | > loss_mel_ce: 3.3591697216033936 (3.3877759309226554) + | > loss: 0.8488432168960571 (0.8571537550745248) + | > current_lr: 5e-06 + | > step_time: 0.3445 (0.24524640895276664) + | > loader_time: 0.0129 (0.011712230625880268) + + + --> TIME: 2024-10-20 04:08:57 -- STEP: 49850/69059 -- GLOBAL_STEP: 49850 + | > loss_text_ce: 0.03504868596792221 (0.040832328216929774) + | > loss_mel_ce: 3.214898109436035 (3.387554803965904) + | > loss: 0.8124867081642151 (0.8570967830518791) + | > current_lr: 5e-06 + | > step_time: 0.396 (0.24526602329916078) + | > loader_time: 0.0165 (0.011712252029564317) + + + --> TIME: 2024-10-20 04:09:16 -- STEP: 49900/69059 -- GLOBAL_STEP: 49900 + | > loss_text_ce: 0.033271487802267075 (0.04082530325558542) + | > loss_mel_ce: 3.4181888103485107 (3.387386095695822) + | > loss: 0.8628650903701782 (0.8570528497425979) + | > current_lr: 5e-06 + | > step_time: 0.3111 (0.24528022200884467) + | > loader_time: 0.0126 (0.01171197580192273) + + + --> TIME: 2024-10-20 04:09:34 -- STEP: 49950/69059 -- GLOBAL_STEP: 49950 + | > loss_text_ce: 0.03541262075304985 (0.040818004290031115) + | > loss_mel_ce: 3.355072021484375 (3.3871628271709873) + | > loss: 0.8476211428642273 (0.8569952078731935) + | > current_lr: 5e-06 + | > step_time: 0.2351 (0.24526382326960444) + | > loader_time: 0.012 (0.011711252244027146) + + + --> TIME: 2024-10-20 04:09:53 -- STEP: 50000/69059 -- GLOBAL_STEP: 50000 + | > loss_text_ce: 0.03583291172981262 (0.04081106553424096) + | > loss_mel_ce: 3.373223066329956 (3.3869503625202007) + | > loss: 0.8522639870643616 (0.8569403570210943) + | > current_lr: 5e-06 + | > step_time: 0.3818 (0.24527167283058166) + | > loader_time: 0.0135 (0.011711140351295448) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_50000.pth + + --> TIME: 2024-10-20 04:10:25 -- STEP: 50050/69059 -- GLOBAL_STEP: 50050 + | > loss_text_ce: 0.03428506478667259 (0.0408036671415874) + | > loss_mel_ce: 3.3590660095214844 (3.3867254692548108) + | > loss: 0.8483377695083618 (0.8568822841067899) + | > current_lr: 5e-06 + | > step_time: 0.2813 (0.24527177494365376) + | > loader_time: 0.0122 (0.011711046397983708) + + + --> TIME: 2024-10-20 04:10:43 -- STEP: 50100/69059 -- GLOBAL_STEP: 50100 + | > loss_text_ce: 0.03425575792789459 (0.04079676697777897) + | > loss_mel_ce: 3.3120765686035156 (3.386509661250944) + | > loss: 0.8365830779075623 (0.8568266070650494) + | > current_lr: 5e-06 + | > step_time: 0.1807 (0.2452585014611661) + | > loader_time: 0.0111 (0.011710844263583101) + + + --> TIME: 2024-10-20 04:11:02 -- STEP: 50150/69059 -- GLOBAL_STEP: 50150 + | > loss_text_ce: 0.03411844000220299 (0.0407896289281757) + | > loss_mel_ce: 3.092456340789795 (3.386334801095772) + | > loss: 0.7816436886787415 (0.856781107514117) + | > current_lr: 5e-06 + | > step_time: 0.3756 (0.2452698687898554) + | > loader_time: 0.0124 (0.011711314825094092) + + + --> TIME: 2024-10-20 04:11:20 -- STEP: 50200/69059 -- GLOBAL_STEP: 50200 + | > loss_text_ce: 0.03439600393176079 (0.04078233552911073) + | > loss_mel_ce: 3.0837581157684326 (3.3861069186939963) + | > loss: 0.7795385122299194 (0.8567223135598636) + | > current_lr: 5e-06 + | > step_time: 0.1828 (0.24525924226677276) + | > loader_time: 0.0108 (0.011711113885104396) + + + --> TIME: 2024-10-20 04:11:39 -- STEP: 50250/69059 -- GLOBAL_STEP: 50250 + | > loss_text_ce: 0.034798841923475266 (0.040775391925329606) + | > loss_mel_ce: 3.261746644973755 (3.385861048010434) + | > loss: 0.8241363763809204 (0.8566591099886167) + | > current_lr: 5e-06 + | > step_time: 0.424 (0.24526503273503697) + | > loader_time: 0.0138 (0.011711030367001943) + + + --> TIME: 2024-10-20 04:11:57 -- STEP: 50300/69059 -- GLOBAL_STEP: 50300 + | > loss_text_ce: 0.03293672949075699 (0.04076733807675764) + | > loss_mel_ce: 3.0566415786743164 (3.3855886353033737) + | > loss: 0.772394597530365 (0.8565889933474267) + | > current_lr: 5e-06 + | > step_time: 0.207 (0.2452546027640462) + | > loader_time: 0.0117 (0.011710784880825801) + + + --> TIME: 2024-10-20 04:12:15 -- STEP: 50350/69059 -- GLOBAL_STEP: 50350 + | > loss_text_ce: 0.032271914184093475 (0.04076015305299302) + | > loss_mel_ce: 3.203976631164551 (3.3854008420765997) + | > loss: 0.809062123298645 (0.8565402487870369) + | > current_lr: 5e-06 + | > step_time: 0.218 (0.24524852269692596) + | > loader_time: 0.015 (0.011710935117234754) + + + --> TIME: 2024-10-20 04:12:33 -- STEP: 50400/69059 -- GLOBAL_STEP: 50400 + | > loss_text_ce: 0.03357847034931183 (0.040752971366209895) + | > loss_mel_ce: 3.2501144409179688 (3.385173696941782) + | > loss: 0.8209232091903687 (0.856481667082935) + | > current_lr: 5e-06 + | > step_time: 0.2036 (0.2452444204829988) + | > loader_time: 0.0118 (0.011710787595264473) + + + --> TIME: 2024-10-20 04:12:51 -- STEP: 50450/69059 -- GLOBAL_STEP: 50450 + | > loss_text_ce: 0.03357934579253197 (0.040745819144917685) + | > loss_mel_ce: 2.9950661659240723 (3.38498193946417) + | > loss: 0.7571613788604736 (0.8564319396574277) + | > current_lr: 5e-06 + | > step_time: 0.2017 (0.24523812694757496) + | > loader_time: 0.0113 (0.011711069263953497) + + + --> TIME: 2024-10-20 04:13:09 -- STEP: 50500/69059 -- GLOBAL_STEP: 50500 + | > loss_text_ce: 0.031626198440790176 (0.04073819104552506) + | > loss_mel_ce: 3.379441261291504 (3.3847675828980757) + | > loss: 0.8527668714523315 (0.8563764434899442) + | > current_lr: 5e-06 + | > step_time: 0.226 (0.24522688606942053) + | > loader_time: 0.0121 (0.01171109856709393) + + + --> TIME: 2024-10-20 04:13:29 -- STEP: 50550/69059 -- GLOBAL_STEP: 50550 + | > loss_text_ce: 0.03302796930074692 (0.04073063199409216) + | > loss_mel_ce: 3.2403595447540283 (3.3845685343218666) + | > loss: 0.8183468580245972 (0.8563247915828271) + | > current_lr: 5e-06 + | > step_time: 0.215 (0.24524326352290182) + | > loader_time: 0.0114 (0.01171116608658363) + + + --> TIME: 2024-10-20 04:13:48 -- STEP: 50600/69059 -- GLOBAL_STEP: 50600 + | > loss_text_ce: 0.033397357910871506 (0.040723283353846375) + | > loss_mel_ce: 3.121100902557373 (3.3843415041308926) + | > loss: 0.7886245846748352 (0.8562661968719355) + | > current_lr: 5e-06 + | > step_time: 0.3197 (0.24525102056533452) + | > loader_time: 0.015 (0.011711097051032425) + + + --> TIME: 2024-10-20 04:14:07 -- STEP: 50650/69059 -- GLOBAL_STEP: 50650 + | > loss_text_ce: 0.03137871250510216 (0.04071633597885487) + | > loss_mel_ce: 3.138559579849243 (3.384149817893189) + | > loss: 0.7924845814704895 (0.8562165384713903) + | > current_lr: 5e-06 + | > step_time: 0.1963 (0.2452552272442651) + | > loader_time: 0.0115 (0.011711048533862319) + + + --> TIME: 2024-10-20 04:14:27 -- STEP: 50700/69059 -- GLOBAL_STEP: 50700 + | > loss_text_ce: 0.03401932865381241 (0.04070941808826326) + | > loss_mel_ce: 2.9423415660858154 (3.3839640075685304) + | > loss: 0.74409019947052 (0.856168356421432) + | > current_lr: 5e-06 + | > step_time: 0.2473 (0.2452772920418538) + | > loader_time: 0.0124 (0.011711156664515365) + + + --> TIME: 2024-10-20 04:14:45 -- STEP: 50750/69059 -- GLOBAL_STEP: 50750 + | > loss_text_ce: 0.03120243176817894 (0.04070236774911964) + | > loss_mel_ce: 3.136863946914673 (3.3837950706293847) + | > loss: 0.7920165657997131 (0.856124359604174) + | > current_lr: 5e-06 + | > step_time: 0.3106 (0.2452784486967942) + | > loader_time: 0.0122 (0.011711139890360692) + + + --> TIME: 2024-10-20 04:15:03 -- STEP: 50800/69059 -- GLOBAL_STEP: 50800 + | > loss_text_ce: 0.03289595618844032 (0.04069491229386664) + | > loss_mel_ce: 3.346381187438965 (3.38356627035327) + | > loss: 0.8448193073272705 (0.8560652956718543) + | > current_lr: 5e-06 + | > step_time: 0.1893 (0.24526712007409945) + | > loader_time: 0.0108 (0.011710938877008071) + + + --> TIME: 2024-10-20 04:15:21 -- STEP: 50850/69059 -- GLOBAL_STEP: 50850 + | > loss_text_ce: 0.0313275009393692 (0.04068776201790063) + | > loss_mel_ce: 2.930344581604004 (3.3833551168676035) + | > loss: 0.740418016910553 (0.8560107197323155) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.24525514774266247) + | > loader_time: 0.0112 (0.011710482450126992) + + + --> TIME: 2024-10-20 04:15:38 -- STEP: 50900/69059 -- GLOBAL_STEP: 50900 + | > loss_text_ce: 0.03399648517370224 (0.04068002035259441) + | > loss_mel_ce: 3.038471221923828 (3.3831279310215026) + | > loss: 0.7681169509887695 (0.8559519878539223) + | > current_lr: 5e-06 + | > step_time: 0.1982 (0.24524237295967657) + | > loader_time: 0.0115 (0.011710077128382212) + + + --> TIME: 2024-10-20 04:15:58 -- STEP: 50950/69059 -- GLOBAL_STEP: 50950 + | > loss_text_ce: 0.03177810460329056 (0.040673434922911596) + | > loss_mel_ce: 3.142207622528076 (3.3829489925915937) + | > loss: 0.7934964299201965 (0.855905606889398) + | > current_lr: 5e-06 + | > step_time: 0.2261 (0.24526677285607126) + | > loader_time: 0.0115 (0.011710146898376322) + + + --> TIME: 2024-10-20 04:16:18 -- STEP: 51000/69059 -- GLOBAL_STEP: 51000 + | > loss_text_ce: 0.03807691112160683 (0.04066666896842071) + | > loss_mel_ce: 3.207940101623535 (3.3827690808726123) + | > loss: 0.8115042448043823 (0.8558589374691841) + | > current_lr: 5e-06 + | > step_time: 0.3723 (0.24527642795151355) + | > loader_time: 0.0132 (0.011709865640191448) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_51000.pth + + --> TIME: 2024-10-20 04:16:49 -- STEP: 51050/69059 -- GLOBAL_STEP: 51050 + | > loss_text_ce: 0.03263728320598602 (0.04065973140620346) + | > loss_mel_ce: 2.9994454383850098 (3.3825920343679097) + | > loss: 0.7580206990242004 (0.8558129414486964) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.2452659096283497) + | > loader_time: 0.0109 (0.011709675377893379) + + + --> TIME: 2024-10-20 04:17:08 -- STEP: 51100/69059 -- GLOBAL_STEP: 51100 + | > loss_text_ce: 0.03500809893012047 (0.040653097450317074) + | > loss_mel_ce: 3.0720748901367188 (3.382395404560908) + | > loss: 0.7767707705497742 (0.8557621255091267) + | > current_lr: 5e-06 + | > step_time: 0.3584 (0.24527271571224682) + | > loader_time: 0.0133 (0.01170985801812485) + + + --> TIME: 2024-10-20 04:17:28 -- STEP: 51150/69059 -- GLOBAL_STEP: 51150 + | > loss_text_ce: 0.03342809900641441 (0.040646377515633506) + | > loss_mel_ce: 3.234286069869995 (3.3822277900410365) + | > loss: 0.8169285655021667 (0.8557185418899467) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.2452849771083858) + | > loader_time: 0.0113 (0.011710058666277702) + + + --> TIME: 2024-10-20 04:17:46 -- STEP: 51200/69059 -- GLOBAL_STEP: 51200 + | > loss_text_ce: 0.03603081777691841 (0.04063927985243879) + | > loss_mel_ce: 3.3945915699005127 (3.3820344473281705) + | > loss: 0.8576555848121643 (0.8556684317940388) + | > current_lr: 5e-06 + | > step_time: 0.4451 (0.24527348777279256) + | > loader_time: 0.0136 (0.011709665548987664) + + + --> TIME: 2024-10-20 04:18:03 -- STEP: 51250/69059 -- GLOBAL_STEP: 51250 + | > loss_text_ce: 0.034655869007110596 (0.040631947555774026) + | > loss_mel_ce: 3.2164711952209473 (3.3818085491087206) + | > loss: 0.8127817511558533 (0.8556101241600231) + | > current_lr: 5e-06 + | > step_time: 0.4088 (0.24526659820835764) + | > loader_time: 0.0133 (0.011709535710404537) + + + --> TIME: 2024-10-20 04:18:22 -- STEP: 51300/69059 -- GLOBAL_STEP: 51300 + | > loss_text_ce: 0.03276805207133293 (0.040625128266355635) + | > loss_mel_ce: 2.756347894668579 (3.381594697364805) + | > loss: 0.6972789764404297 (0.8555549564022311) + | > current_lr: 5e-06 + | > step_time: 0.1726 (0.24526353030176887) + | > loader_time: 0.0108 (0.011709350233654275) + + + --> TIME: 2024-10-20 04:18:41 -- STEP: 51350/69059 -- GLOBAL_STEP: 51350 + | > loss_text_ce: 0.03131513670086861 (0.040618223782697124) + | > loss_mel_ce: 3.1041693687438965 (3.3813946354725637) + | > loss: 0.7838711142539978 (0.8555032148101206) + | > current_lr: 5e-06 + | > step_time: 0.1865 (0.24527317836430892) + | > loader_time: 0.0105 (0.011709182102236777) + + + --> TIME: 2024-10-20 04:18:59 -- STEP: 51400/69059 -- GLOBAL_STEP: 51400 + | > loss_text_ce: 0.030190858989953995 (0.040611552221642) + | > loss_mel_ce: 3.1173300743103027 (3.381190235206572) + | > loss: 0.7868802547454834 (0.8554504468596408) + | > current_lr: 5e-06 + | > step_time: 0.1576 (0.2452726932722307) + | > loader_time: 0.0107 (0.011709352165808451) + + + --> TIME: 2024-10-20 04:19:20 -- STEP: 51450/69059 -- GLOBAL_STEP: 51450 + | > loss_text_ce: 0.034700535237789154 (0.040604680002937064) + | > loss_mel_ce: 3.202824592590332 (3.38105154397998) + | > loss: 0.8093813061714172 (0.8554140560029208) + | > current_lr: 5e-06 + | > step_time: 0.1736 (0.24529969436434199) + | > loader_time: 0.0126 (0.011709703995952083) + + + --> TIME: 2024-10-20 04:19:38 -- STEP: 51500/69059 -- GLOBAL_STEP: 51500 + | > loss_text_ce: 0.033844489604234695 (0.0405977699926244) + | > loss_mel_ce: 3.5398616790771484 (3.380863015327622) + | > loss: 0.8934265375137329 (0.8553651963368212) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.24529662877610586) + | > loader_time: 0.0106 (0.01170967680273702) + + + --> TIME: 2024-10-20 04:19:58 -- STEP: 51550/69059 -- GLOBAL_STEP: 51550 + | > loss_text_ce: 0.03178941830992699 (0.040590788255744095) + | > loss_mel_ce: 3.0690977573394775 (3.380664888751016) + | > loss: 0.7752217650413513 (0.855313919256546) + | > current_lr: 5e-06 + | > step_time: 0.1676 (0.24530931187878524) + | > loader_time: 0.0112 (0.011709728314966722) + + + --> TIME: 2024-10-20 04:20:16 -- STEP: 51600/69059 -- GLOBAL_STEP: 51600 + | > loss_text_ce: 0.033642373979091644 (0.04058383675479786) + | > loss_mel_ce: 3.364285945892334 (3.3804566440055366) + | > loss: 0.8494820594787598 (0.8552601201959367) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.2453030139531276) + | > loader_time: 0.0114 (0.0117094845365184) + + + --> TIME: 2024-10-20 04:20:34 -- STEP: 51650/69059 -- GLOBAL_STEP: 51650 + | > loss_text_ce: 0.032616451382637024 (0.04057679107464931) + | > loss_mel_ce: 3.5233242511749268 (3.3801938836410517) + | > loss: 0.8889851570129395 (0.8551926686837629) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.24529291575783566) + | > loader_time: 0.0101 (0.01170928854448403) + + + --> TIME: 2024-10-20 04:20:52 -- STEP: 51700/69059 -- GLOBAL_STEP: 51700 + | > loss_text_ce: 0.03315712511539459 (0.04057003540756049) + | > loss_mel_ce: 3.080742120742798 (3.3800112867170715) + | > loss: 0.7784748077392578 (0.8551453305374501) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.2452948361439216) + | > loader_time: 0.0101 (0.01170913175180754) + + + --> TIME: 2024-10-20 04:21:11 -- STEP: 51750/69059 -- GLOBAL_STEP: 51750 + | > loss_text_ce: 0.031032128259539604 (0.04056304478303334) + | > loss_mel_ce: 3.123940944671631 (3.3797911920869814) + | > loss: 0.788743257522583 (0.8550885592232588) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24530009197382535) + | > loader_time: 0.0104 (0.011708984425678323) + + + --> TIME: 2024-10-20 04:21:29 -- STEP: 51800/69059 -- GLOBAL_STEP: 51800 + | > loss_text_ce: 0.03291334956884384 (0.04055569166493722) + | > loss_mel_ce: 3.027677297592163 (3.3795167059235425) + | > loss: 0.7651476860046387 (0.8550180994002984) + | > current_lr: 5e-06 + | > step_time: 0.1574 (0.2452899897696889) + | > loader_time: 0.0126 (0.011708827313308988) + + + --> TIME: 2024-10-20 04:21:48 -- STEP: 51850/69059 -- GLOBAL_STEP: 51850 + | > loss_text_ce: 0.03503655269742012 (0.04054883124696514) + | > loss_mel_ce: 3.0479896068573 (3.3793380221888154) + | > loss: 0.7707565426826477 (0.854971713365677) + | > current_lr: 5e-06 + | > step_time: 0.2266 (0.24528797881237682) + | > loader_time: 0.0113 (0.011708564933033759) + + + --> TIME: 2024-10-20 04:22:05 -- STEP: 51900/69059 -- GLOBAL_STEP: 51900 + | > loss_text_ce: 0.03235979005694389 (0.04054195996549692) + | > loss_mel_ce: 3.284180164337158 (3.379085346370746) + | > loss: 0.829135000705719 (0.85490682658892) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.24527051783251166) + | > loader_time: 0.0109 (0.011708189044614266) + + + --> TIME: 2024-10-20 04:22:23 -- STEP: 51950/69059 -- GLOBAL_STEP: 51950 + | > loss_text_ce: 0.03593410551548004 (0.04053537585439967) + | > loss_mel_ce: 3.173872232437134 (3.378870241061424) + | > loss: 0.8024516105651855 (0.8548514042311394) + | > current_lr: 5e-06 + | > step_time: 0.2165 (0.24526353719948116) + | > loader_time: 0.0141 (0.011707829988496114) + + + --> TIME: 2024-10-20 04:22:40 -- STEP: 52000/69059 -- GLOBAL_STEP: 52000 + | > loss_text_ce: 0.03427787125110626 (0.040528772037583205) + | > loss_mel_ce: 3.2638320922851562 (3.3786223014226158) + | > loss: 0.8245275020599365 (0.8547877683708309) + | > current_lr: 5e-06 + | > step_time: 0.2273 (0.2452393694107349) + | > loader_time: 0.0115 (0.01170749894930764) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_52000.pth + + --> TIME: 2024-10-20 04:23:14 -- STEP: 52050/69059 -- GLOBAL_STEP: 52050 + | > loss_text_ce: 0.033976826816797256 (0.040521995963726805) + | > loss_mel_ce: 3.12153959274292 (3.378423808997446) + | > loss: 0.7888790965080261 (0.8547364512472408) + | > current_lr: 5e-06 + | > step_time: 0.3353 (0.2452573642538329) + | > loader_time: 0.0134 (0.011707759769230802) + + + --> TIME: 2024-10-20 04:23:33 -- STEP: 52100/69059 -- GLOBAL_STEP: 52100 + | > loss_text_ce: 0.033560123294591904 (0.04051560129773624) + | > loss_mel_ce: 3.2689425945281982 (3.3782537129012105) + | > loss: 0.8256256580352783 (0.8546923285562575) + | > current_lr: 5e-06 + | > step_time: 0.182 (0.24526378776198843) + | > loader_time: 0.0105 (0.011707846143652955) + + + --> TIME: 2024-10-20 04:23:52 -- STEP: 52150/69059 -- GLOBAL_STEP: 52150 + | > loss_text_ce: 0.03483515977859497 (0.04050900251687603) + | > loss_mel_ce: 3.092552900314331 (3.3780554975553367) + | > loss: 0.7818470001220703 (0.8546411250250231) + | > current_lr: 5e-06 + | > step_time: 0.2429 (0.2452780716439793) + | > loader_time: 0.0112 (0.011708124705723333) + + + --> TIME: 2024-10-20 04:24:12 -- STEP: 52200/69059 -- GLOBAL_STEP: 52200 + | > loss_text_ce: 0.033462610095739365 (0.040502285044532696) + | > loss_mel_ce: 3.014758348464966 (3.3778333310742776) + | > loss: 0.7620552182197571 (0.8545839040375313) + | > current_lr: 5e-06 + | > step_time: 0.375 (0.24528762795459266) + | > loader_time: 0.0135 (0.011708358941863757) + + + --> TIME: 2024-10-20 04:24:32 -- STEP: 52250/69059 -- GLOBAL_STEP: 52250 + | > loss_text_ce: 0.03414470702409744 (0.04049536415823996) + | > loss_mel_ce: 3.30934739112854 (3.3776603356365786) + | > loss: 0.835873007774353 (0.8545389249575774) + | > current_lr: 5e-06 + | > step_time: 0.514 (0.2453099613691631) + | > loader_time: 0.0153 (0.011708691528539317) + + + --> TIME: 2024-10-20 04:24:50 -- STEP: 52300/69059 -- GLOBAL_STEP: 52300 + | > loss_text_ce: 0.03371994197368622 (0.04048893533265928) + | > loss_mel_ce: 3.228498697280884 (3.3774175070394126) + | > loss: 0.815554678440094 (0.8544766105999234) + | > current_lr: 5e-06 + | > step_time: 0.3008 (0.24530192281730318) + | > loader_time: 0.013 (0.011708647224688827) + + + --> TIME: 2024-10-20 04:25:07 -- STEP: 52350/69059 -- GLOBAL_STEP: 52350 + | > loss_text_ce: 0.035044074058532715 (0.0404819938803038) + | > loss_mel_ce: 3.082386016845703 (3.3771949379673947) + | > loss: 0.7793575525283813 (0.8544192329678184) + | > current_lr: 5e-06 + | > step_time: 0.2269 (0.2452806610683088) + | > loader_time: 0.0112 (0.011708396335044337) + + + --> TIME: 2024-10-20 04:25:24 -- STEP: 52400/69059 -- GLOBAL_STEP: 52400 + | > loss_text_ce: 0.03171497583389282 (0.04047487334909843) + | > loss_mel_ce: 3.160442590713501 (3.3769508938479804) + | > loss: 0.7980393767356873 (0.8543564418026518) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24526765942573547) + | > loader_time: 0.0109 (0.011708070253597849) + + + --> TIME: 2024-10-20 04:25:43 -- STEP: 52450/69059 -- GLOBAL_STEP: 52450 + | > loss_text_ce: 0.03342539072036743 (0.04046807396109271) + | > loss_mel_ce: 3.1637136936187744 (3.3767333378805278) + | > loss: 0.7992847561836243 (0.8543003529611383) + | > current_lr: 5e-06 + | > step_time: 0.2049 (0.24526829806819658) + | > loader_time: 0.011 (0.011707891908796524) + + + --> TIME: 2024-10-20 04:26:02 -- STEP: 52500/69059 -- GLOBAL_STEP: 52500 + | > loss_text_ce: 0.03299004212021828 (0.04046149638930192) + | > loss_mel_ce: 3.1321258544921875 (3.3765572108313986) + | > loss: 0.7912789583206177 (0.8542546768052246) + | > current_lr: 5e-06 + | > step_time: 0.2475 (0.2452728131112598) + | > loader_time: 0.0114 (0.01170785533814201) + + + --> TIME: 2024-10-20 04:26:19 -- STEP: 52550/69059 -- GLOBAL_STEP: 52550 + | > loss_text_ce: 0.035075388848781586 (0.0404547786221016) + | > loss_mel_ce: 3.1987311840057373 (3.3763712700835478) + | > loss: 0.8084516525268555 (0.8542065121807684) + | > current_lr: 5e-06 + | > step_time: 0.4061 (0.24525551027394157) + | > loader_time: 0.0129 (0.011707344713038633) + + + --> TIME: 2024-10-20 04:26:37 -- STEP: 52600/69059 -- GLOBAL_STEP: 52600 + | > loss_text_ce: 0.034810688346624374 (0.04044800533658058) + | > loss_mel_ce: 3.163743734359741 (3.3761494400836503) + | > loss: 0.7996386289596558 (0.8541493613627029) + | > current_lr: 5e-06 + | > step_time: 0.2302 (0.24524725252684532) + | > loader_time: 0.0113 (0.01170721820552084) + + + --> TIME: 2024-10-20 04:26:56 -- STEP: 52650/69059 -- GLOBAL_STEP: 52650 + | > loss_text_ce: 0.03355686366558075 (0.04044141495184597) + | > loss_mel_ce: 3.5584640502929688 (3.3759571075394) + | > loss: 0.8980052471160889 (0.8540996306268587) + | > current_lr: 5e-06 + | > step_time: 0.5931 (0.2452563617258896) + | > loader_time: 0.0153 (0.011707033938944941) + + + --> TIME: 2024-10-20 04:27:14 -- STEP: 52700/69059 -- GLOBAL_STEP: 52700 + | > loss_text_ce: 0.03366302698850632 (0.040434158359259606) + | > loss_mel_ce: 3.4308269023895264 (3.3757542057598173) + | > loss: 0.8661224842071533 (0.8540470910332469) + | > current_lr: 5e-06 + | > step_time: 0.3954 (0.24525012121707257) + | > loader_time: 0.0131 (0.011706738689140284) + + + --> TIME: 2024-10-20 04:27:34 -- STEP: 52750/69059 -- GLOBAL_STEP: 52750 + | > loss_text_ce: 0.034907419234514236 (0.04042802362871341) + | > loss_mel_ce: 3.0976204872131348 (3.3755077040161403) + | > loss: 0.7831319570541382 (0.8539839319131959) + | > current_lr: 5e-06 + | > step_time: 0.1715 (0.24525768168842624) + | > loader_time: 0.0102 (0.011706498204814293) + + + --> TIME: 2024-10-20 04:27:51 -- STEP: 52800/69059 -- GLOBAL_STEP: 52800 + | > loss_text_ce: 0.03630217909812927 (0.040421203154308) + | > loss_mel_ce: 3.2907779216766357 (3.375294062531339) + | > loss: 0.8317700028419495 (0.8539288164234983) + | > current_lr: 5e-06 + | > step_time: 0.3431 (0.24524586187619152) + | > loader_time: 0.0131 (0.011706017612507828) + + + --> TIME: 2024-10-20 04:28:11 -- STEP: 52850/69059 -- GLOBAL_STEP: 52850 + | > loss_text_ce: 0.03210144117474556 (0.04041441046050364) + | > loss_mel_ce: 3.0094549655914307 (3.3751136849889525) + | > loss: 0.7603890895843506 (0.853882023867506) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24525829926006276) + | > loader_time: 0.0104 (0.011705794212608351) + + + --> TIME: 2024-10-20 04:28:29 -- STEP: 52900/69059 -- GLOBAL_STEP: 52900 + | > loss_text_ce: 0.03438175097107887 (0.04040790651743518) + | > loss_mel_ce: 3.2534775733947754 (3.374956868312293) + | > loss: 0.8219648599624634 (0.8538411937116007) + | > current_lr: 5e-06 + | > step_time: 0.2869 (0.24525085012042905) + | > loader_time: 0.0122 (0.011705270827605257) + + + --> TIME: 2024-10-20 04:28:47 -- STEP: 52950/69059 -- GLOBAL_STEP: 52950 + | > loss_text_ce: 0.03414159640669823 (0.04040076967849358) + | > loss_mel_ce: 2.929591178894043 (3.3747248408161608) + | > loss: 0.7409331798553467 (0.8537814026299918) + | > current_lr: 5e-06 + | > step_time: 0.2317 (0.2452490901766913) + | > loader_time: 0.0108 (0.01170471476427892) + + + --> TIME: 2024-10-20 04:29:05 -- STEP: 53000/69059 -- GLOBAL_STEP: 53000 + | > loss_text_ce: 0.03270232677459717 (0.0403940137011468) + | > loss_mel_ce: 2.9528284072875977 (3.3745261212339774) + | > loss: 0.7463827133178711 (0.853730033738434) + | > current_lr: 5e-06 + | > step_time: 0.1525 (0.24523645978153877) + | > loader_time: 0.0105 (0.011704212494616216) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_53000.pth + + --> TIME: 2024-10-20 04:29:40 -- STEP: 53050/69059 -- GLOBAL_STEP: 53050 + | > loss_text_ce: 0.03569817170500755 (0.04038744577764006) + | > loss_mel_ce: 3.345479965209961 (3.374354207311213) + | > loss: 0.8452945351600647 (0.8536854132765546) + | > current_lr: 5e-06 + | > step_time: 0.5933 (0.24525045751738841) + | > loader_time: 0.0155 (0.011704318305437101) + + + --> TIME: 2024-10-20 04:29:58 -- STEP: 53100/69059 -- GLOBAL_STEP: 53100 + | > loss_text_ce: 0.03628283366560936 (0.04038063076354523) + | > loss_mel_ce: 3.19817852973938 (3.374138693715203) + | > loss: 0.8086153268814087 (0.8536298311250591) + | > current_lr: 5e-06 + | > step_time: 0.1519 (0.24524610584963064) + | > loader_time: 0.0103 (0.011703985922754127) + + + --> TIME: 2024-10-20 04:30:17 -- STEP: 53150/69059 -- GLOBAL_STEP: 53150 + | > loss_text_ce: 0.03325388953089714 (0.04037393397387844) + | > loss_mel_ce: 3.2102744579315186 (3.3739231232753313) + | > loss: 0.8108820915222168 (0.8535742643175555) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.2452466186082038) + | > loader_time: 0.0114 (0.011704041753752751) + + + --> TIME: 2024-10-20 04:30:36 -- STEP: 53200/69059 -- GLOBAL_STEP: 53200 + | > loss_text_ce: 0.03485369309782982 (0.04036755562546307) + | > loss_mel_ce: 3.4412972927093506 (3.3737095453774852) + | > loss: 0.8690377473831177 (0.8535192752556702) + | > current_lr: 5e-06 + | > step_time: 0.2075 (0.24525530024130543) + | > loader_time: 0.0111 (0.011703951815913473) + + + --> TIME: 2024-10-20 04:30:54 -- STEP: 53250/69059 -- GLOBAL_STEP: 53250 + | > loss_text_ce: 0.0349157452583313 (0.040360902891541886) + | > loss_mel_ce: 3.14823317527771 (3.37348373730528) + | > loss: 0.7957872152328491 (0.85346116005423) + | > current_lr: 5e-06 + | > step_time: 0.3913 (0.2452398498830661) + | > loader_time: 0.0135 (0.011703575196960142) + + + --> TIME: 2024-10-20 04:31:12 -- STEP: 53300/69059 -- GLOBAL_STEP: 53300 + | > loss_text_ce: 0.03224902227520943 (0.040354190699049684) + | > loss_mel_ce: 2.8708300590515137 (3.3732686355458252) + | > loss: 0.7257697582244873 (0.8534057065660027) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24523943145100663) + | > loader_time: 0.0119 (0.011703707420356257) + + + --> TIME: 2024-10-20 04:31:31 -- STEP: 53350/69059 -- GLOBAL_STEP: 53350 + | > loss_text_ce: 0.03177093341946602 (0.04034763851738022) + | > loss_mel_ce: 3.088947296142578 (3.373062544794075) + | > loss: 0.7801795601844788 (0.8533525458305049) + | > current_lr: 5e-06 + | > step_time: 0.2023 (0.24524203012228682) + | > loader_time: 0.0117 (0.011703862451233346) + + + --> TIME: 2024-10-20 04:31:49 -- STEP: 53400/69059 -- GLOBAL_STEP: 53400 + | > loss_text_ce: 0.03431194648146629 (0.04034096237505672) + | > loss_mel_ce: 3.3238894939422607 (3.3728446381234782) + | > loss: 0.8395503759384155 (0.8532964001266705) + | > current_lr: 5e-06 + | > step_time: 0.2453 (0.24523536913403857) + | > loader_time: 0.0117 (0.011703792350569) + + + --> TIME: 2024-10-20 04:32:06 -- STEP: 53450/69059 -- GLOBAL_STEP: 53450 + | > loss_text_ce: 0.031641848385334015 (0.04033425595186129) + | > loss_mel_ce: 3.348299264907837 (3.37258669718499) + | > loss: 0.84498530626297 (0.8532302382875545) + | > current_lr: 5e-06 + | > step_time: 0.1581 (0.24520874849538918) + | > loader_time: 0.0101 (0.011703389215067885) + + + --> TIME: 2024-10-20 04:32:24 -- STEP: 53500/69059 -- GLOBAL_STEP: 53500 + | > loss_text_ce: 0.033928945660591125 (0.04032780092815318) + | > loss_mel_ce: 3.0910658836364746 (3.372404825518048) + | > loss: 0.7812486886978149 (0.8531831566139925) + | > current_lr: 5e-06 + | > step_time: 0.1995 (0.24520062914964194) + | > loader_time: 0.0132 (0.01170318273740392) + + + --> TIME: 2024-10-20 04:32:41 -- STEP: 53550/69059 -- GLOBAL_STEP: 53550 + | > loss_text_ce: 0.032432135194540024 (0.04032112660277737) + | > loss_mel_ce: 2.921562433242798 (3.372166906618588) + | > loss: 0.7384986281394958 (0.8531220083098683) + | > current_lr: 5e-06 + | > step_time: 0.1957 (0.24517608435993213) + | > loader_time: 0.0114 (0.01170305412801817) + + + --> TIME: 2024-10-20 04:33:00 -- STEP: 53600/69059 -- GLOBAL_STEP: 53600 + | > loss_text_ce: 0.0323355607688427 (0.040314869684611475) + | > loss_mel_ce: 3.353855609893799 (3.371984742092536) + | > loss: 0.8465477824211121 (0.853074902952163) + | > current_lr: 5e-06 + | > step_time: 0.1647 (0.24519311642913677) + | > loader_time: 0.0115 (0.011703178531198339) + + + --> TIME: 2024-10-20 04:33:20 -- STEP: 53650/69059 -- GLOBAL_STEP: 53650 + | > loss_text_ce: 0.03148995712399483 (0.040308350460066894) + | > loss_mel_ce: 3.0092310905456543 (3.3718083595229498) + | > loss: 0.7601802349090576 (0.8530291775010624) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24520427985604634) + | > loader_time: 0.0104 (0.011702976986724178) + + + --> TIME: 2024-10-20 04:33:38 -- STEP: 53700/69059 -- GLOBAL_STEP: 53700 + | > loss_text_ce: 0.03323904424905777 (0.040301586632586796) + | > loss_mel_ce: 3.0281729698181152 (3.371590377929252) + | > loss: 0.7653530240058899 (0.8529729911424159) + | > current_lr: 5e-06 + | > step_time: 0.1715 (0.24520487725401724) + | > loader_time: 0.0116 (0.01170272702389365) + + + --> TIME: 2024-10-20 04:33:57 -- STEP: 53750/69059 -- GLOBAL_STEP: 53750 + | > loss_text_ce: 0.033133942633867264 (0.0402952655241581) + | > loss_mel_ce: 3.1161372661590576 (3.3714401852851528) + | > loss: 0.7873178124427795 (0.8529338627083365) + | > current_lr: 5e-06 + | > step_time: 0.3754 (0.24520410635305007) + | > loader_time: 0.0126 (0.011702515540012072) + + + --> TIME: 2024-10-20 04:34:17 -- STEP: 53800/69059 -- GLOBAL_STEP: 53800 + | > loss_text_ce: 0.03390064090490341 (0.04028887686541174) + | > loss_mel_ce: 3.0223641395568848 (3.3712055592155648) + | > loss: 0.764066219329834 (0.8528736090250183) + | > current_lr: 5e-06 + | > step_time: 0.2051 (0.24522035707771556) + | > loader_time: 0.0113 (0.011704365183429625) + + + --> TIME: 2024-10-20 04:34:34 -- STEP: 53850/69059 -- GLOBAL_STEP: 53850 + | > loss_text_ce: 0.03422397002577782 (0.04028215959775333) + | > loss_mel_ce: 3.1523709297180176 (3.3709736441326066) + | > loss: 0.7966487407684326 (0.8528139509342294) + | > current_lr: 5e-06 + | > step_time: 0.1944 (0.24521204387023698) + | > loader_time: 0.011 (0.011704098680225235) + + + --> TIME: 2024-10-20 04:34:54 -- STEP: 53900/69059 -- GLOBAL_STEP: 53900 + | > loss_text_ce: 0.03249535337090492 (0.04027586157373159) + | > loss_mel_ce: 3.2615084648132324 (3.3708485572855476) + | > loss: 0.82350093126297 (0.8527811047163885) + | > current_lr: 5e-06 + | > step_time: 0.2469 (0.2452231392435771) + | > loader_time: 0.0124 (0.011704114191164976) + + + --> TIME: 2024-10-20 04:35:13 -- STEP: 53950/69059 -- GLOBAL_STEP: 53950 + | > loss_text_ce: 0.030750909820199013 (0.04026957416238833) + | > loss_mel_ce: 3.035092353820801 (3.3706207723767645) + | > loss: 0.7664608359336853 (0.852722586636416) + | > current_lr: 5e-06 + | > step_time: 0.1852 (0.24523119538000493) + | > loader_time: 0.0103 (0.011703927280507318) + + + --> TIME: 2024-10-20 04:35:30 -- STEP: 54000/69059 -- GLOBAL_STEP: 54000 + | > loss_text_ce: 0.035593267530202866 (0.040262774816196285) + | > loss_mel_ce: 3.319506883621216 (3.3703670527846565) + | > loss: 0.8387750387191772 (0.8526574569013392) + | > current_lr: 5e-06 + | > step_time: 0.259 (0.24520767592942272) + | > loader_time: 0.0122 (0.011703476561440341) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_54000.pth + + --> TIME: 2024-10-20 04:36:02 -- STEP: 54050/69059 -- GLOBAL_STEP: 54050 + | > loss_text_ce: 0.03534732386469841 (0.04025613867408033) + | > loss_mel_ce: 3.1927828788757324 (3.370189639893409) + | > loss: 0.8070325255393982 (0.8526114446404471) + | > current_lr: 5e-06 + | > step_time: 0.2117 (0.24520849889125348) + | > loader_time: 0.0112 (0.011703413514270462) + + + --> TIME: 2024-10-20 04:36:21 -- STEP: 54100/69059 -- GLOBAL_STEP: 54100 + | > loss_text_ce: 0.031966693699359894 (0.04024978715337608) + | > loss_mel_ce: 3.1375224590301514 (3.3700203585756743) + | > loss: 0.7923722863197327 (0.8525675364298662) + | > current_lr: 5e-06 + | > step_time: 0.1517 (0.245213826209472) + | > loader_time: 0.0106 (0.011705669488571907) + + + --> TIME: 2024-10-20 04:36:39 -- STEP: 54150/69059 -- GLOBAL_STEP: 54150 + | > loss_text_ce: 0.03441983461380005 (0.04024335987614767) + | > loss_mel_ce: 3.17679500579834 (3.3698224156671013) + | > loss: 0.8028036952018738 (0.8525164438846047) + | > current_lr: 5e-06 + | > step_time: 0.2106 (0.24520402670126762) + | > loader_time: 0.0108 (0.011705427086144141) + + + --> TIME: 2024-10-20 04:36:58 -- STEP: 54200/69059 -- GLOBAL_STEP: 54200 + | > loss_text_ce: 0.03333786875009537 (0.04023693672075429) + | > loss_mel_ce: 3.1436924934387207 (3.3696058078357494) + | > loss: 0.7942575812339783 (0.8524606861333573) + | > current_lr: 5e-06 + | > step_time: 0.2 (0.24520201277468917) + | > loader_time: 0.0105 (0.011705248553814463) + + + --> TIME: 2024-10-20 04:37:17 -- STEP: 54250/69059 -- GLOBAL_STEP: 54250 + | > loss_text_ce: 0.034234918653964996 (0.04023086514880115) + | > loss_mel_ce: 3.041102409362793 (3.3694425086755198) + | > loss: 0.7688343524932861 (0.8524183434495187) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.24521219165204308) + | > loader_time: 0.0108 (0.011705323399486609) + + + --> TIME: 2024-10-20 04:37:37 -- STEP: 54300/69059 -- GLOBAL_STEP: 54300 + | > loss_text_ce: 0.03403138741850853 (0.04022428769614789) + | > loss_mel_ce: 3.2253849506378174 (3.3692831478109952) + | > loss: 0.814854085445404 (0.8523768588719455) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.24523214699813675) + | > loader_time: 0.0113 (0.011705755967900657) + + + --> TIME: 2024-10-20 04:37:56 -- STEP: 54350/69059 -- GLOBAL_STEP: 54350 + | > loss_text_ce: 0.03320569545030594 (0.04021763556274397) + | > loss_mel_ce: 3.095731735229492 (3.3690938372028403) + | > loss: 0.7822343707084656 (0.8523278681883166) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.2452330964622743) + | > loader_time: 0.0104 (0.011705845812645745) + + + --> TIME: 2024-10-20 04:38:14 -- STEP: 54400/69059 -- GLOBAL_STEP: 54400 + | > loss_text_ce: 0.034485824406147 (0.04021116797747174) + | > loss_mel_ce: 2.9868996143341064 (3.368879764150155) + | > loss: 0.7553463578224182 (0.8522727330313894) + | > current_lr: 5e-06 + | > step_time: 0.1572 (0.2452260746938341) + | > loader_time: 0.0104 (0.011705685810131164) + + + --> TIME: 2024-10-20 04:38:32 -- STEP: 54450/69059 -- GLOBAL_STEP: 54450 + | > loss_text_ce: 0.03145945072174072 (0.04020485109489621) + | > loss_mel_ce: 3.1496992111206055 (3.368656622691353) + | > loss: 0.7952896356582642 (0.8522153684466784) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.2452287304390451) + | > loader_time: 0.0103 (0.011705452868213756) + + + --> TIME: 2024-10-20 04:38:51 -- STEP: 54500/69059 -- GLOBAL_STEP: 54500 + | > loss_text_ce: 0.035020340234041214 (0.040198586927709244) + | > loss_mel_ce: 2.9938387870788574 (3.3684318991275943) + | > loss: 0.7572147846221924 (0.8521576215149074) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.24522820069374293) + | > loader_time: 0.0104 (0.01170506701775645) + + + --> TIME: 2024-10-20 04:39:09 -- STEP: 54550/69059 -- GLOBAL_STEP: 54550 + | > loss_text_ce: 0.034546978771686554 (0.04019213830138735) + | > loss_mel_ce: 3.0205249786376953 (3.3682616452751035) + | > loss: 0.7637680172920227 (0.8521134458978068) + | > current_lr: 5e-06 + | > step_time: 0.1844 (0.24522097188982803) + | > loader_time: 0.0105 (0.011704713076653534) + + + --> TIME: 2024-10-20 04:39:27 -- STEP: 54600/69059 -- GLOBAL_STEP: 54600 + | > loss_text_ce: 0.033809807151556015 (0.04018552286737979) + | > loss_mel_ce: 2.9312899112701416 (3.3680362219819058) + | > loss: 0.7412749528884888 (0.8520554362133754) + | > current_lr: 5e-06 + | > step_time: 0.2268 (0.24521233733757075) + | > loader_time: 0.012 (0.011704739131333606) + + + --> TIME: 2024-10-20 04:39:45 -- STEP: 54650/69059 -- GLOBAL_STEP: 54650 + | > loss_text_ce: 0.033411767333745956 (0.040178893307957875) + | > loss_mel_ce: 3.428288221359253 (3.367802314671032) + | > loss: 0.8654249906539917 (0.8519953019959282) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.24520497861741775) + | > loader_time: 0.0103 (0.011704550483089332) + + + --> TIME: 2024-10-20 04:40:04 -- STEP: 54700/69059 -- GLOBAL_STEP: 54700 + | > loss_text_ce: 0.03466162085533142 (0.04017272830019623) + | > loss_mel_ce: 3.217663288116455 (3.367630395636464) + | > loss: 0.8130812048912048 (0.8519507809814203) + | > current_lr: 5e-06 + | > step_time: 0.1648 (0.2452075592429808) + | > loader_time: 0.0107 (0.011704443599449848) + + + --> TIME: 2024-10-20 04:40:23 -- STEP: 54750/69059 -- GLOBAL_STEP: 54750 + | > loss_text_ce: 0.03400971367955208 (0.040166545631882274) + | > loss_mel_ce: 3.4399237632751465 (3.3674494521302014) + | > loss: 0.8684833645820618 (0.8519039994353037) + | > current_lr: 5e-06 + | > step_time: 0.2054 (0.24522382962213804) + | > loader_time: 0.0108 (0.011704506464744791) + + + --> TIME: 2024-10-20 04:40:42 -- STEP: 54800/69059 -- GLOBAL_STEP: 54800 + | > loss_text_ce: 0.032723650336265564 (0.04016048836888842) + | > loss_mel_ce: 3.1203832626342773 (3.3672711507116633) + | > loss: 0.788276731967926 (0.8518579097657751) + | > current_lr: 5e-06 + | > step_time: 0.1714 (0.24523247935911166) + | > loader_time: 0.0118 (0.01170434819520823) + + + --> TIME: 2024-10-20 04:41:00 -- STEP: 54850/69059 -- GLOBAL_STEP: 54850 + | > loss_text_ce: 0.03153610602021217 (0.04015406229946652) + | > loss_mel_ce: 2.8155875205993652 (3.3670625743804954) + | > loss: 0.7117809057235718 (0.8518041591635601) + | > current_lr: 5e-06 + | > step_time: 0.185 (0.24521458694037243) + | > loader_time: 0.011 (0.011703962558165594) + + + --> TIME: 2024-10-20 04:41:19 -- STEP: 54900/69059 -- GLOBAL_STEP: 54900 + | > loss_text_ce: 0.03534102067351341 (0.04014770869185963) + | > loss_mel_ce: 3.321704626083374 (3.366905105057511) + | > loss: 0.8392614126205444 (0.8517632034274406) + | > current_lr: 5e-06 + | > step_time: 0.1816 (0.24521809395110888) + | > loader_time: 0.0115 (0.011703779306568062) + + + --> TIME: 2024-10-20 04:41:36 -- STEP: 54950/69059 -- GLOBAL_STEP: 54950 + | > loss_text_ce: 0.03187432512640953 (0.04014088621408752) + | > loss_mel_ce: 3.5297586917877197 (3.3666810859041343) + | > loss: 0.8904082775115967 (0.851705493015849) + | > current_lr: 5e-06 + | > step_time: 0.3783 (0.24520040610142466) + | > loader_time: 0.0171 (0.011703389031546435) + + + --> TIME: 2024-10-20 04:41:53 -- STEP: 55000/69059 -- GLOBAL_STEP: 55000 + | > loss_text_ce: 0.033265385776758194 (0.04013465938933818) + | > loss_mel_ce: 3.4939403533935547 (3.366496560326474) + | > loss: 0.881801426410675 (0.8516578049150388) + | > current_lr: 5e-06 + | > step_time: 0.1652 (0.24518934760093689) + | > loader_time: 0.0114 (0.011702834562821802) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_55000.pth + + --> TIME: 2024-10-20 04:42:27 -- STEP: 55050/69059 -- GLOBAL_STEP: 55050 + | > loss_text_ce: 0.032284654676914215 (0.040128812762137506) + | > loss_mel_ce: 3.201439380645752 (3.3663386302596168) + | > loss: 0.8084310293197632 (0.8516168607396507) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.24520524319034614) + | > loader_time: 0.0103 (0.011702853518112675) + + + --> TIME: 2024-10-20 04:42:46 -- STEP: 55100/69059 -- GLOBAL_STEP: 55100 + | > loss_text_ce: 0.03259224444627762 (0.040122451500260094) + | > loss_mel_ce: 3.151676893234253 (3.36618478710983) + | > loss: 0.7960672974586487 (0.8515768096364785) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.24520850693032875) + | > loader_time: 0.0108 (0.011703415046802645) + + + --> TIME: 2024-10-20 04:43:04 -- STEP: 55150/69059 -- GLOBAL_STEP: 55150 + | > loss_text_ce: 0.03475535660982132 (0.040116628088049756) + | > loss_mel_ce: 3.186663866043091 (3.3660097498474237) + | > loss: 0.8053548336029053 (0.8515315944685905) + | > current_lr: 5e-06 + | > step_time: 0.326 (0.24520973897693596) + | > loader_time: 0.0125 (0.011703374617119195) + + + --> TIME: 2024-10-20 04:43:25 -- STEP: 55200/69059 -- GLOBAL_STEP: 55200 + | > loss_text_ce: 0.033733878284692764 (0.04011041825711405) + | > loss_mel_ce: 3.015512466430664 (3.365876031353838) + | > loss: 0.762311577796936 (0.8514966123885874) + | > current_lr: 5e-06 + | > step_time: 0.1654 (0.24523762452861536) + | > loader_time: 0.0111 (0.011703663088273291) + + + --> TIME: 2024-10-20 04:43:43 -- STEP: 55250/69059 -- GLOBAL_STEP: 55250 + | > loss_text_ce: 0.03224186971783638 (0.040103825065614426) + | > loss_mel_ce: 2.817399024963379 (3.365674181877742) + | > loss: 0.7124102115631104 (0.8514445017204033) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.2452356751817384) + | > loader_time: 0.0115 (0.011703800909120012) + + + --> TIME: 2024-10-20 04:44:01 -- STEP: 55300/69059 -- GLOBAL_STEP: 55300 + | > loss_text_ce: 0.03147905692458153 (0.040097146658815766) + | > loss_mel_ce: 3.2882907390594482 (3.365458538208889) + | > loss: 0.8299424648284912 (0.8513889211998287) + | > current_lr: 5e-06 + | > step_time: 0.1816 (0.2452159716035315) + | > loader_time: 0.0106 (0.011703533574426779) + + + --> TIME: 2024-10-20 04:44:18 -- STEP: 55350/69059 -- GLOBAL_STEP: 55350 + | > loss_text_ce: 0.032597385346889496 (0.04009066493183914) + | > loss_mel_ce: 2.9958460330963135 (3.365266087195183) + | > loss: 0.7571108341217041 (0.8513391880162048) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.24520779669554038) + | > loader_time: 0.0105 (0.01170327322700588) + + + --> TIME: 2024-10-20 04:44:36 -- STEP: 55400/69059 -- GLOBAL_STEP: 55400 + | > loss_text_ce: 0.033932216465473175 (0.04008462231959221) + | > loss_mel_ce: 2.7836437225341797 (3.3650714366960885) + | > loss: 0.7043939828872681 (0.8512890147409723) + | > current_lr: 5e-06 + | > step_time: 0.1635 (0.24519590272800157) + | > loader_time: 0.0107 (0.01170342178120938) + + + --> TIME: 2024-10-20 04:44:54 -- STEP: 55450/69059 -- GLOBAL_STEP: 55450 + | > loss_text_ce: 0.030094465240836143 (0.04007843928319174) + | > loss_mel_ce: 2.823890209197998 (3.364872588169677) + | > loss: 0.7134961485862732 (0.8512377568501635) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.24518878535605207) + | > loader_time: 0.0113 (0.011703558501302906) + + + --> TIME: 2024-10-20 04:45:12 -- STEP: 55500/69059 -- GLOBAL_STEP: 55500 + | > loss_text_ce: 0.032232556492090225 (0.04007245995000221) + | > loss_mel_ce: 3.56060528755188 (3.364709337320398) + | > loss: 0.8982094526290894 (0.851195449306085) + | > current_lr: 5e-06 + | > step_time: 0.1643 (0.2451759091153875) + | > loader_time: 0.0106 (0.01170331266334463) + + + --> TIME: 2024-10-20 04:45:31 -- STEP: 55550/69059 -- GLOBAL_STEP: 55550 + | > loss_text_ce: 0.03117513656616211 (0.04006609655480736) + | > loss_mel_ce: 3.0819473266601562 (3.3645217770311078) + | > loss: 0.7782806158065796 (0.8511469683831718) + | > current_lr: 5e-06 + | > step_time: 0.1846 (0.24517598896387136) + | > loader_time: 0.011 (0.011703419020109388) + + + --> TIME: 2024-10-20 04:45:50 -- STEP: 55600/69059 -- GLOBAL_STEP: 55600 + | > loss_text_ce: 0.03355726972222328 (0.040060409445569684) + | > loss_mel_ce: 3.14335560798645 (3.364359319630271) + | > loss: 0.794228196144104 (0.8511049322537388) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.24518951576390713) + | > loader_time: 0.011 (0.011703736915005177) + + + --> TIME: 2024-10-20 04:46:08 -- STEP: 55650/69059 -- GLOBAL_STEP: 55650 + | > loss_text_ce: 0.03218385949730873 (0.04005392030728234) + | > loss_mel_ce: 3.130352020263672 (3.3641935029419554) + | > loss: 0.7906339764595032 (0.8510618557966416) + | > current_lr: 5e-06 + | > step_time: 0.1952 (0.24518263951358985) + | > loader_time: 0.0119 (0.011703630449101375) + + + --> TIME: 2024-10-20 04:46:26 -- STEP: 55700/69059 -- GLOBAL_STEP: 55700 + | > loss_text_ce: 0.03271954879164696 (0.04004780825919668) + | > loss_mel_ce: 2.8464226722717285 (3.3639654253372226) + | > loss: 0.7197855710983276 (0.8510033083828826) + | > current_lr: 5e-06 + | > step_time: 0.1631 (0.24517462645227647) + | > loader_time: 0.0102 (0.011703346708732599) + + + --> TIME: 2024-10-20 04:46:44 -- STEP: 55750/69059 -- GLOBAL_STEP: 55750 + | > loss_text_ce: 0.03577284887433052 (0.04004159794964947) + | > loss_mel_ce: 3.2577810287475586 (3.363767019536982) + | > loss: 0.8233884572982788 (0.8509521543551996) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.2451669637825457) + | > loader_time: 0.0102 (0.011703223523537655) + + + --> TIME: 2024-10-20 04:47:03 -- STEP: 55800/69059 -- GLOBAL_STEP: 55800 + | > loss_text_ce: 0.032353807240724564 (0.040035825797145846) + | > loss_mel_ce: 2.8987510204315186 (3.363558788560182) + | > loss: 0.7327762246131897 (0.8508986535732472) + | > current_lr: 5e-06 + | > step_time: 0.1516 (0.2451690010913384) + | > loader_time: 0.0105 (0.011703021560518525) + + + --> TIME: 2024-10-20 04:47:23 -- STEP: 55850/69059 -- GLOBAL_STEP: 55850 + | > loss_text_ce: 0.03125801682472229 (0.04002987151666957) + | > loss_mel_ce: 2.9893743991851807 (3.3633943643928377) + | > loss: 0.7551581263542175 (0.8508560589604227) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.24518791345580945) + | > loader_time: 0.0107 (0.011703039359704853) + + + --> TIME: 2024-10-20 04:47:40 -- STEP: 55900/69059 -- GLOBAL_STEP: 55900 + | > loss_text_ce: 0.03272872790694237 (0.0400237693543639) + | > loss_mel_ce: 3.358527421951294 (3.3632170444908054) + | > loss: 0.8478140234947205 (0.8508102034466859) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.24517748164789407) + | > loader_time: 0.0107 (0.01170271154806309) + + + --> TIME: 2024-10-20 04:48:00 -- STEP: 55950/69059 -- GLOBAL_STEP: 55950 + | > loss_text_ce: 0.030578361824154854 (0.040017356526262544) + | > loss_mel_ce: 3.0062389373779297 (3.363008179847831) + | > loss: 0.7592043280601501 (0.8507563840746355) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.24519144495860928) + | > loader_time: 0.0107 (0.011702810024128502) + + + --> TIME: 2024-10-20 04:48:18 -- STEP: 56000/69059 -- GLOBAL_STEP: 56000 + | > loss_text_ce: 0.03388398885726929 (0.0400110341641448) + | > loss_mel_ce: 3.373960256576538 (3.3628303358895284) + | > loss: 0.851961076259613 (0.8507103424902481) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.2451855636366776) + | > loader_time: 0.0114 (0.01170234661442891) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_56000.pth + + --> TIME: 2024-10-20 04:48:50 -- STEP: 56050/69059 -- GLOBAL_STEP: 56050 + | > loss_text_ce: 0.036481063812971115 (0.040004509936429435) + | > loss_mel_ce: 2.9673612117767334 (3.3626395474875275) + | > loss: 0.7509605884552002 (0.8506610143354817) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.2451658119674669) + | > loader_time: 0.0108 (0.011702079432655062) + + + --> TIME: 2024-10-20 04:49:08 -- STEP: 56100/69059 -- GLOBAL_STEP: 56100 + | > loss_text_ce: 0.033828962594270706 (0.03999810713517582) + | > loss_mel_ce: 2.9698288440704346 (3.362405579629838) + | > loss: 0.750914454460144 (0.8506009216678342) + | > current_lr: 5e-06 + | > step_time: 0.163 (0.24516469961394174) + | > loader_time: 0.0105 (0.011701774057432484) + + + --> TIME: 2024-10-20 04:49:28 -- STEP: 56150/69059 -- GLOBAL_STEP: 56150 + | > loss_text_ce: 0.03104802593588829 (0.0399920259507323) + | > loss_mel_ce: 3.036832332611084 (3.3622188885596063) + | > loss: 0.7669700980186462 (0.8505527286060355) + | > current_lr: 5e-06 + | > step_time: 0.1842 (0.2451793786785163) + | > loader_time: 0.0109 (0.011701665727134677) + + + --> TIME: 2024-10-20 04:49:47 -- STEP: 56200/69059 -- GLOBAL_STEP: 56200 + | > loss_text_ce: 0.0336233489215374 (0.039986257237411546) + | > loss_mel_ce: 2.9826033115386963 (3.3619953908869227) + | > loss: 0.7540566921234131 (0.8504954120113765) + | > current_lr: 5e-06 + | > step_time: 0.2364 (0.24519285033181892) + | > loader_time: 0.0123 (0.01170195604558515) + + + --> TIME: 2024-10-20 04:50:05 -- STEP: 56250/69059 -- GLOBAL_STEP: 56250 + | > loss_text_ce: 0.03292305767536163 (0.03998018991145771) + | > loss_mel_ce: 3.0208094120025635 (3.3618067624409838) + | > loss: 0.7634330987930298 (0.8504467380714424) + | > current_lr: 5e-06 + | > step_time: 0.206 (0.245188268699646) + | > loader_time: 0.0109 (0.011701819932725675) + + + --> TIME: 2024-10-20 04:50:23 -- STEP: 56300/69059 -- GLOBAL_STEP: 56300 + | > loss_text_ce: 0.03015570528805256 (0.03997423215681506) + | > loss_mel_ce: 2.775956630706787 (3.361604213714584) + | > loss: 0.7015280723571777 (0.8503946114475739) + | > current_lr: 5e-06 + | > step_time: 0.1576 (0.24517535092352127) + | > loader_time: 0.0106 (0.011701805176573982) + + + --> TIME: 2024-10-20 04:50:42 -- STEP: 56350/69059 -- GLOBAL_STEP: 56350 + | > loss_text_ce: 0.03424102067947388 (0.03996824298576196) + | > loss_mel_ce: 3.176528215408325 (3.3614220209595427) + | > loss: 0.8026922941207886 (0.8503475659657294) + | > current_lr: 5e-06 + | > step_time: 0.3555 (0.24518374542060317) + | > loader_time: 0.0109 (0.011701673924023829) + + + --> TIME: 2024-10-20 04:51:02 -- STEP: 56400/69059 -- GLOBAL_STEP: 56400 + | > loss_text_ce: 0.031044138595461845 (0.03996192707075745) + | > loss_mel_ce: 3.0185744762420654 (3.3612503665896862) + | > loss: 0.7624046802520752 (0.8503030733943838) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.24519642470998967) + | > loader_time: 0.0112 (0.011701692752804296) + + + --> TIME: 2024-10-20 04:51:22 -- STEP: 56450/69059 -- GLOBAL_STEP: 56450 + | > loss_text_ce: 0.031223611906170845 (0.03995595307217198) + | > loss_mel_ce: 2.8424971103668213 (3.3611043408784806) + | > loss: 0.7184301614761353 (0.850265073466872) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.24521855065841608) + | > loader_time: 0.0118 (0.011701899254818297) + + + --> TIME: 2024-10-20 04:51:38 -- STEP: 56500/69059 -- GLOBAL_STEP: 56500 + | > loss_text_ce: 0.03020968846976757 (0.03994983088683318) + | > loss_mel_ce: 3.0174014568328857 (3.3608858057207613) + | > loss: 0.7619028091430664 (0.8502089091328401) + | > current_lr: 5e-06 + | > step_time: 0.1872 (0.24518943531323323) + | > loader_time: 0.0118 (0.011701615789295277) + + + --> TIME: 2024-10-20 04:51:57 -- STEP: 56550/69059 -- GLOBAL_STEP: 56550 + | > loss_text_ce: 0.03223492205142975 (0.039943934701033454) + | > loss_mel_ce: 2.8686363697052 (3.3607237783772512) + | > loss: 0.7252178192138672 (0.8501669282506366) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.2451940263424375) + | > loader_time: 0.0126 (0.011701621077526555) + + + --> TIME: 2024-10-20 04:52:16 -- STEP: 56600/69059 -- GLOBAL_STEP: 56600 + | > loss_text_ce: 0.033958639949560165 (0.03993805790935384) + | > loss_mel_ce: 3.315645456314087 (3.360582052262833) + | > loss: 0.8374010324478149 (0.8501300275241536) + | > current_lr: 5e-06 + | > step_time: 0.1814 (0.24519253817548178) + | > loader_time: 0.0109 (0.011701539653771317) + + + --> TIME: 2024-10-20 04:52:35 -- STEP: 56650/69059 -- GLOBAL_STEP: 56650 + | > loss_text_ce: 0.030599769204854965 (0.03993222243436097) + | > loss_mel_ce: 3.276308298110962 (3.3603994394590084) + | > loss: 0.826727032661438 (0.8500829154542356) + | > current_lr: 5e-06 + | > step_time: 0.176 (0.24519653040018108) + | > loader_time: 0.0103 (0.011701395854630075) + + + --> TIME: 2024-10-20 04:52:54 -- STEP: 56700/69059 -- GLOBAL_STEP: 56700 + | > loss_text_ce: 0.033331066370010376 (0.039926489749380796) + | > loss_mel_ce: 3.2101571559906006 (3.3602062910910693) + | > loss: 0.8108720779418945 (0.850033195193273) + | > current_lr: 5e-06 + | > step_time: 0.1571 (0.24520920194225546) + | > loader_time: 0.0102 (0.01170115750512957) + + + --> TIME: 2024-10-20 04:53:12 -- STEP: 56750/69059 -- GLOBAL_STEP: 56750 + | > loss_text_ce: 0.032407328486442566 (0.03992019192615087) + | > loss_mel_ce: 3.0601131916046143 (3.3600316929627994) + | > loss: 0.7731301188468933 (0.849987971201868) + | > current_lr: 5e-06 + | > step_time: 0.2236 (0.24519868048697316) + | > loader_time: 0.0115 (0.011700849075149315) + + + --> TIME: 2024-10-20 04:53:30 -- STEP: 56800/69059 -- GLOBAL_STEP: 56800 + | > loss_text_ce: 0.03086971677839756 (0.039913904364220286) + | > loss_mel_ce: 3.086242437362671 (3.3598407456824324) + | > loss: 0.779278039932251 (0.8499386624928938) + | > current_lr: 5e-06 + | > step_time: 0.1559 (0.24519122335692525) + | > loader_time: 0.011 (0.01170054810567639) + + + --> TIME: 2024-10-20 04:53:46 -- STEP: 56850/69059 -- GLOBAL_STEP: 56850 + | > loss_text_ce: 0.03181243687868118 (0.039907856759145384) + | > loss_mel_ce: 3.01667857170105 (3.3596398463999746) + | > loss: 0.7621227502822876 (0.8498869257677836) + | > current_lr: 5e-06 + | > step_time: 0.1718 (0.24516977333666048) + | > loader_time: 0.0109 (0.011700171321342132) + + + --> TIME: 2024-10-20 04:54:06 -- STEP: 56900/69059 -- GLOBAL_STEP: 56900 + | > loss_text_ce: 0.030953340232372284 (0.03990186119740139) + | > loss_mel_ce: 3.07212233543396 (3.3594717870780846) + | > loss: 0.7757689356803894 (0.8498434120510927) + | > current_lr: 5e-06 + | > step_time: 0.2076 (0.24517891079763537) + | > loader_time: 0.0148 (0.011700349222796643) + + + --> TIME: 2024-10-20 04:54:24 -- STEP: 56950/69059 -- GLOBAL_STEP: 56950 + | > loss_text_ce: 0.03136871010065079 (0.03989593952567093) + | > loss_mel_ce: 3.0227251052856445 (3.359272924537074) + | > loss: 0.7635234594345093 (0.8497922159957726) + | > current_lr: 5e-06 + | > step_time: 0.1675 (0.2451743779211739) + | > loader_time: 0.01 (0.011700194210625178) + + + --> TIME: 2024-10-20 04:54:42 -- STEP: 57000/69059 -- GLOBAL_STEP: 57000 + | > loss_text_ce: 0.035854604095220566 (0.039889815211034144) + | > loss_mel_ce: 2.9904959201812744 (3.3591372012380813) + | > loss: 0.7565876245498657 (0.8497567540909122) + | > current_lr: 5e-06 + | > step_time: 0.1823 (0.24516421976842379) + | > loader_time: 0.0108 (0.01170003020135978) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_57000.pth + + --> TIME: 2024-10-20 04:55:15 -- STEP: 57050/69059 -- GLOBAL_STEP: 57050 + | > loss_text_ce: 0.029563812538981438 (0.03988385557243509) + | > loss_mel_ce: 2.98856520652771 (3.3589577293604798) + | > loss: 0.7545322775840759 (0.8497103962095872) + | > current_lr: 5e-06 + | > step_time: 0.1939 (0.24516983414197782) + | > loader_time: 0.0112 (0.011700343366885354) + + + --> TIME: 2024-10-20 04:55:34 -- STEP: 57100/69059 -- GLOBAL_STEP: 57100 + | > loss_text_ce: 0.0350571945309639 (0.03987793595241348) + | > loss_mel_ce: 2.871194362640381 (3.3587920696555686) + | > loss: 0.7265629172325134 (0.8496675013786029) + | > current_lr: 5e-06 + | > step_time: 0.1657 (0.24517238140523748) + | > loader_time: 0.0125 (0.011700377756576385) + + + --> TIME: 2024-10-20 04:55:51 -- STEP: 57150/69059 -- GLOBAL_STEP: 57150 + | > loss_text_ce: 0.032414671033620834 (0.039871545839444765) + | > loss_mel_ce: 3.7412405014038086 (3.3585885439352783) + | > loss: 0.9434137940406799 (0.8496150224217183) + | > current_lr: 5e-06 + | > step_time: 0.2173 (0.24515231713222393) + | > loader_time: 0.0137 (0.011700320915495818) + + + --> TIME: 2024-10-20 04:56:10 -- STEP: 57200/69059 -- GLOBAL_STEP: 57200 + | > loss_text_ce: 0.03358237072825432 (0.0398656333290309) + | > loss_mel_ce: 3.1830570697784424 (3.3583643544595407) + | > loss: 0.8041598796844482 (0.8495574969261684) + | > current_lr: 5e-06 + | > step_time: 0.1834 (0.24514892260094623) + | > loader_time: 0.0111 (0.011700435105737253) + + + --> TIME: 2024-10-20 04:56:28 -- STEP: 57250/69059 -- GLOBAL_STEP: 57250 + | > loss_text_ce: 0.034129660576581955 (0.03985995686929378) + | > loss_mel_ce: 3.3100433349609375 (3.3582083827314264) + | > loss: 0.8360432386398315 (0.8495170848775646) + | > current_lr: 5e-06 + | > step_time: 0.2353 (0.2451484196841977) + | > loader_time: 0.0119 (0.011700533879375854) + + + --> TIME: 2024-10-20 04:56:46 -- STEP: 57300/69059 -- GLOBAL_STEP: 57300 + | > loss_text_ce: 0.033210624009370804 (0.03985426152355749) + | > loss_mel_ce: 2.8405373096466064 (3.3579856370922423) + | > loss: 0.7184369564056396 (0.8494599746305497) + | > current_lr: 5e-06 + | > step_time: 0.151 (0.2451468342885921) + | > loader_time: 0.0104 (0.011700502631968) + + + --> TIME: 2024-10-20 04:57:05 -- STEP: 57350/69059 -- GLOBAL_STEP: 57350 + | > loss_text_ce: 0.03153051435947418 (0.039848555167836024) + | > loss_mel_ce: 2.9826416969299316 (3.3578246674832672) + | > loss: 0.7535430788993835 (0.8494183056398181) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.24514933816432538) + | > loader_time: 0.0106 (0.011700288814986989) + + + --> TIME: 2024-10-20 04:57:23 -- STEP: 57400/69059 -- GLOBAL_STEP: 57400 + | > loss_text_ce: 0.03259694203734398 (0.03984289637782953) + | > loss_mel_ce: 3.3409221172332764 (3.357627515211323) + | > loss: 0.8433797359466553 (0.8493676028735554) + | > current_lr: 5e-06 + | > step_time: 0.3196 (0.2451422724906576) + | > loader_time: 0.0141 (0.011700155303037935) + + + --> TIME: 2024-10-20 04:57:43 -- STEP: 57450/69059 -- GLOBAL_STEP: 57450 + | > loss_text_ce: 0.033128056675195694 (0.03983714383743893) + | > loss_mel_ce: 2.897057056427002 (3.3574762728983942) + | > loss: 0.7325462698936462 (0.8493283541571904) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.24516376688337202) + | > loader_time: 0.0109 (0.01170020733842441) + + + --> TIME: 2024-10-20 04:58:04 -- STEP: 57500/69059 -- GLOBAL_STEP: 57500 + | > loss_text_ce: 0.0332963727414608 (0.03983134351871533) + | > loss_mel_ce: 3.275425672531128 (3.3573176046661555) + | > loss: 0.8271805047988892 (0.8492872370160152) + | > current_lr: 5e-06 + | > step_time: 0.165 (0.24518955442179804) + | > loader_time: 0.0105 (0.011700274032095184) + + + --> TIME: 2024-10-20 04:58:22 -- STEP: 57550/69059 -- GLOBAL_STEP: 57550 + | > loss_text_ce: 0.03359999507665634 (0.03982577714635305) + | > loss_mel_ce: 3.3169801235198975 (3.357109670763307) + | > loss: 0.8376450538635254 (0.8492338619482825) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.24518664055964307) + | > loader_time: 0.0108 (0.011700009297123152) + + + --> TIME: 2024-10-20 04:58:43 -- STEP: 57600/69059 -- GLOBAL_STEP: 57600 + | > loss_text_ce: 0.0301259346306324 (0.03982021553707) + | > loss_mel_ce: 3.1730971336364746 (3.356960089964156) + | > loss: 0.8008057475090027 (0.8491950763451563) + | > current_lr: 5e-06 + | > step_time: 0.1517 (0.24521706790559822) + | > loader_time: 0.0099 (0.011700078965061221) + + + --> TIME: 2024-10-20 04:59:01 -- STEP: 57650/69059 -- GLOBAL_STEP: 57650 + | > loss_text_ce: 0.03274239972233772 (0.03981452807373699) + | > loss_mel_ce: 3.105032205581665 (3.3567963705947736) + | > loss: 0.7844436764717102 (0.8491527246363979) + | > current_lr: 5e-06 + | > step_time: 0.1858 (0.2452041998389278) + | > loader_time: 0.0107 (0.011700143925748675) + + + --> TIME: 2024-10-20 04:59:19 -- STEP: 57700/69059 -- GLOBAL_STEP: 57700 + | > loss_text_ce: 0.031843122094869614 (0.03980884207754735) + | > loss_mel_ce: 3.0190229415893555 (3.3565721297016027) + | > loss: 0.76271653175354 (0.8490952429180549) + | > current_lr: 5e-06 + | > step_time: 0.1562 (0.24519573481078785) + | > loader_time: 0.0112 (0.01169970715273189) + + + --> TIME: 2024-10-20 04:59:37 -- STEP: 57750/69059 -- GLOBAL_STEP: 57750 + | > loss_text_ce: 0.03525139018893242 (0.039802884333577) + | > loss_mel_ce: 3.2022080421447754 (3.356412179971655) + | > loss: 0.8093648552894592 (0.8490537660503807) + | > current_lr: 5e-06 + | > step_time: 0.2765 (0.24519392339698165) + | > loader_time: 0.012 (0.011699519413374184) + + + --> TIME: 2024-10-20 04:59:56 -- STEP: 57800/69059 -- GLOBAL_STEP: 57800 + | > loss_text_ce: 0.033420126885175705 (0.03979725191053791) + | > loss_mel_ce: 2.851548910140991 (3.3562127676966953) + | > loss: 0.7212422490119934 (0.849002504874679) + | > current_lr: 5e-06 + | > step_time: 0.1734 (0.2451978721379409) + | > loader_time: 0.0105 (0.01169929944932665) + + + --> TIME: 2024-10-20 05:00:14 -- STEP: 57850/69059 -- GLOBAL_STEP: 57850 + | > loss_text_ce: 0.0310045313090086 (0.03979164230184017) + | > loss_mel_ce: 3.5932230949401855 (3.3560424535276416) + | > loss: 0.9060568809509277 (0.8489585239287593) + | > current_lr: 5e-06 + | > step_time: 0.2833 (0.24519679369963546) + | > loader_time: 0.0116 (0.011698819022809193) + + + --> TIME: 2024-10-20 05:00:34 -- STEP: 57900/69059 -- GLOBAL_STEP: 57900 + | > loss_text_ce: 0.031930018216371536 (0.03978616504960471) + | > loss_mel_ce: 3.5112876892089844 (3.3558803914206687) + | > loss: 0.8858044147491455 (0.8489166390895851) + | > current_lr: 5e-06 + | > step_time: 0.2309 (0.24521400471425428) + | > loader_time: 0.0113 (0.011698599294059616) + + + --> TIME: 2024-10-20 05:00:53 -- STEP: 57950/69059 -- GLOBAL_STEP: 57950 + | > loss_text_ce: 0.03314772620797157 (0.039780477225824985) + | > loss_mel_ce: 3.139294147491455 (3.3556575856114024) + | > loss: 0.7931104898452759 (0.8488595156828022) + | > current_lr: 5e-06 + | > step_time: 0.2107 (0.24521089769829132) + | > loader_time: 0.0114 (0.011698194360609586) + + + --> TIME: 2024-10-20 05:01:11 -- STEP: 58000/69059 -- GLOBAL_STEP: 58000 + | > loss_text_ce: 0.03135760873556137 (0.039774661113636) + | > loss_mel_ce: 2.6406242847442627 (3.3554281521418847) + | > loss: 0.6679954528808594 (0.8488007032840423) + | > current_lr: 5e-06 + | > step_time: 0.1629 (0.2452100645303726) + | > loader_time: 0.0101 (0.011697784362168128) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_58000.pth + + --> TIME: 2024-10-20 05:01:44 -- STEP: 58050/69059 -- GLOBAL_STEP: 58050 + | > loss_text_ce: 0.03042108193039894 (0.03976878787077188) + | > loss_mel_ce: 3.178280830383301 (3.355265352884092) + | > loss: 0.8021754622459412 (0.848758535158214) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.2452097188676872) + | > loader_time: 0.0103 (0.011697685975231641) + + + --> TIME: 2024-10-20 05:02:02 -- STEP: 58100/69059 -- GLOBAL_STEP: 58100 + | > loss_text_ce: 0.032688580453395844 (0.03976309909246389) + | > loss_mel_ce: 3.1280832290649414 (3.3550737460837157) + | > loss: 0.7901929616928101 (0.8487092112633127) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24521141080150505) + | > loader_time: 0.0105 (0.011697568380483848) + + + --> TIME: 2024-10-20 05:02:23 -- STEP: 58150/69059 -- GLOBAL_STEP: 58150 + | > loss_text_ce: 0.035820167511701584 (0.0397577167530431) + | > loss_mel_ce: 3.2689671516418457 (3.354931063274818) + | > loss: 0.8261968493461609 (0.8486721949780293) + | > current_lr: 5e-06 + | > step_time: 0.3928 (0.24523283033461174) + | > loader_time: 0.0127 (0.011697628930633867) + + + --> TIME: 2024-10-20 05:02:42 -- STEP: 58200/69059 -- GLOBAL_STEP: 58200 + | > loss_text_ce: 0.03427452594041824 (0.03975202353623137) + | > loss_mel_ce: 3.0636613368988037 (3.3547546486510296) + | > loss: 0.7744839787483215 (0.8486266680189841) + | > current_lr: 5e-06 + | > step_time: 0.2072 (0.24524189663506865) + | > loader_time: 0.012 (0.011697634524086477) + + + --> TIME: 2024-10-20 05:03:00 -- STEP: 58250/69059 -- GLOBAL_STEP: 58250 + | > loss_text_ce: 0.031475674360990524 (0.03974584139279108) + | > loss_mel_ce: 3.52388072013855 (3.354564267821563) + | > loss: 0.8888391256332397 (0.8485775272784835) + | > current_lr: 5e-06 + | > step_time: 0.3548 (0.24522911786419127) + | > loader_time: 0.0136 (0.011697651457888866) + + + --> TIME: 2024-10-20 05:03:18 -- STEP: 58300/69059 -- GLOBAL_STEP: 58300 + | > loss_text_ce: 0.03362784534692764 (0.03973982063350481) + | > loss_mel_ce: 3.3578410148620605 (3.354392683456923) + | > loss: 0.8478671908378601 (0.8485331259982262) + | > current_lr: 5e-06 + | > step_time: 0.2502 (0.2452263095039446) + | > loader_time: 0.0129 (0.011697529364204715) + + + --> TIME: 2024-10-20 05:03:37 -- STEP: 58350/69059 -- GLOBAL_STEP: 58350 + | > loss_text_ce: 0.03362380713224411 (0.039734118041975346) + | > loss_mel_ce: 3.1620213985443115 (3.354199375816141) + | > loss: 0.7989112734794617 (0.8484833734384853) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24522979512414875) + | > loader_time: 0.0103 (0.011697501641823456) + + + --> TIME: 2024-10-20 05:03:55 -- STEP: 58400/69059 -- GLOBAL_STEP: 58400 + | > loss_text_ce: 0.03649675473570824 (0.03972837286701453) + | > loss_mel_ce: 3.3170111179351807 (3.353993854073614) + | > loss: 0.8383769392967224 (0.8484305567080035) + | > current_lr: 5e-06 + | > step_time: 0.3944 (0.24522760837453686) + | > loader_time: 0.0131 (0.011697534861629937) + + + --> TIME: 2024-10-20 05:04:14 -- STEP: 58450/69059 -- GLOBAL_STEP: 58450 + | > loss_text_ce: 0.033623531460762024 (0.039722539132175244) + | > loss_mel_ce: 3.3394041061401367 (3.353835857862877) + | > loss: 0.8432568907737732 (0.848389599221298) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.24522610154082572) + | > loader_time: 0.0105 (0.011697498073528913) + + + --> TIME: 2024-10-20 05:04:33 -- STEP: 58500/69059 -- GLOBAL_STEP: 58500 + | > loss_text_ce: 0.03541770577430725 (0.03971701118837133) + | > loss_mel_ce: 3.091752767562866 (3.3536270299324475) + | > loss: 0.7817926406860352 (0.8483360102523094) + | > current_lr: 5e-06 + | > step_time: 0.1904 (0.24523650715086195) + | > loader_time: 0.0111 (0.011697428116431584) + + + --> TIME: 2024-10-20 05:04:52 -- STEP: 58550/69059 -- GLOBAL_STEP: 58550 + | > loss_text_ce: 0.03333238139748573 (0.039711447564246674) + | > loss_mel_ce: 3.0773892402648926 (3.3534254893922535) + | > loss: 0.7776803970336914 (0.8482842342147052) + | > current_lr: 5e-06 + | > step_time: 0.2956 (0.24523360215731912) + | > loader_time: 0.0129 (0.011697233116188975) + + + --> TIME: 2024-10-20 05:05:10 -- STEP: 58600/69059 -- GLOBAL_STEP: 58600 + | > loss_text_ce: 0.03251020982861519 (0.03970601735601578) + | > loss_mel_ce: 2.6752331256866455 (3.353232725537256) + | > loss: 0.6769358515739441 (0.8482346857000951) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.24522644427042364) + | > loader_time: 0.0146 (0.01169711744419135) + + + --> TIME: 2024-10-20 05:05:28 -- STEP: 58650/69059 -- GLOBAL_STEP: 58650 + | > loss_text_ce: 0.032229941338300705 (0.03970031421180552) + | > loss_mel_ce: 3.136931896209717 (3.353077876939783) + | > loss: 0.792290449142456 (0.8481945477676318) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.2452306421536822) + | > loader_time: 0.0113 (0.01169697723730139) + + + --> TIME: 2024-10-20 05:05:49 -- STEP: 58700/69059 -- GLOBAL_STEP: 58700 + | > loss_text_ce: 0.03361516073346138 (0.03969508583113197) + | > loss_mel_ce: 3.2306578159332275 (3.352964890783025) + | > loss: 0.8160682320594788 (0.8481649941317682) + | > current_lr: 5e-06 + | > step_time: 0.3135 (0.24525864689272958) + | > loader_time: 0.012 (0.011697119533304849) + + + --> TIME: 2024-10-20 05:06:07 -- STEP: 58750/69059 -- GLOBAL_STEP: 58750 + | > loss_text_ce: 0.0354321226477623 (0.03968908738141949) + | > loss_mel_ce: 3.3457069396972656 (3.352771463763445) + | > loss: 0.8452847599983215 (0.8481151377657634) + | > current_lr: 5e-06 + | > step_time: 0.3537 (0.24525262786784072) + | > loader_time: 0.0124 (0.011696938112948784) + + + --> TIME: 2024-10-20 05:06:25 -- STEP: 58800/69059 -- GLOBAL_STEP: 58800 + | > loss_text_ce: 0.0341934897005558 (0.0396833397382802) + | > loss_mel_ce: 2.8576602935791016 (3.352603422602003) + | > loss: 0.7229634523391724 (0.8480716905626318) + | > current_lr: 5e-06 + | > step_time: 0.1729 (0.24524358739252805) + | > loader_time: 0.0107 (0.011696721493792353) + + + --> TIME: 2024-10-20 05:06:43 -- STEP: 58850/69059 -- GLOBAL_STEP: 58850 + | > loss_text_ce: 0.03195195272564888 (0.039677328844107164) + | > loss_mel_ce: 2.9394004344940186 (3.352398529871081) + | > loss: 0.7428380846977234 (0.8480189646546283) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.2452288237930661) + | > loader_time: 0.0103 (0.011696587714889625) + + + --> TIME: 2024-10-20 05:07:03 -- STEP: 58900/69059 -- GLOBAL_STEP: 58900 + | > loss_text_ce: 0.03391280025243759 (0.03967179780816436) + | > loss_mel_ce: 2.8591792583465576 (3.3522325430499125) + | > loss: 0.7232730388641357 (0.8479760851914699) + | > current_lr: 5e-06 + | > step_time: 0.1632 (0.24524795759715937) + | > loader_time: 0.0101 (0.011696702507031977) + + + --> TIME: 2024-10-20 05:07:20 -- STEP: 58950/69059 -- GLOBAL_STEP: 58950 + | > loss_text_ce: 0.032973360270261765 (0.039666016439545766) + | > loss_mel_ce: 2.897721529006958 (3.352041995203814) + | > loss: 0.732673704624176 (0.8479270028835446) + | > current_lr: 5e-06 + | > step_time: 0.1855 (0.24523545050843476) + | > loader_time: 0.0112 (0.011696548777378829) + + + --> TIME: 2024-10-20 05:07:40 -- STEP: 59000/69059 -- GLOBAL_STEP: 59000 + | > loss_text_ce: 0.035614658147096634 (0.03966082821552734) + | > loss_mel_ce: 3.2191338539123535 (3.351869224548325) + | > loss: 0.8136871457099915 (0.8478825131685054) + | > current_lr: 5e-06 + | > step_time: 0.3141 (0.2452533000970291) + | > loader_time: 0.0122 (0.011696581105054416) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_59000.pth + + --> TIME: 2024-10-20 05:08:12 -- STEP: 59050/69059 -- GLOBAL_STEP: 59050 + | > loss_text_ce: 0.03598926588892937 (0.039655253608141997) + | > loss_mel_ce: 3.1581099033355713 (3.3516935901011835) + | > loss: 0.798524796962738 (0.8478372109047351) + | > current_lr: 5e-06 + | > step_time: 0.2159 (0.24523902852286938) + | > loader_time: 0.0107 (0.011696404877807185) + + + --> TIME: 2024-10-20 05:08:30 -- STEP: 59100/69059 -- GLOBAL_STEP: 59100 + | > loss_text_ce: 0.03438212722539902 (0.03964988648837821) + | > loss_mel_ce: 3.416808605194092 (3.3515064601768954) + | > loss: 0.8627976775169373 (0.8477890866419998) + | > current_lr: 5e-06 + | > step_time: 0.2483 (0.24522698598464734) + | > loader_time: 0.0117 (0.011696060579238589) + + + --> TIME: 2024-10-20 05:08:48 -- STEP: 59150/69059 -- GLOBAL_STEP: 59150 + | > loss_text_ce: 0.032238345593214035 (0.03964444479186256) + | > loss_mel_ce: 2.931293487548828 (3.3513183831160744) + | > loss: 0.740882933139801 (0.8477407069510566) + | > current_lr: 5e-06 + | > step_time: 0.2234 (0.24522272759741479) + | > loader_time: 0.0109 (0.011695875662016023) + + + --> TIME: 2024-10-20 05:09:05 -- STEP: 59200/69059 -- GLOBAL_STEP: 59200 + | > loss_text_ce: 0.031130051240324974 (0.03963894548618211) + | > loss_mel_ce: 2.877208948135376 (3.351129116786478) + | > loss: 0.727084755897522 (0.8476920155405603) + | > current_lr: 5e-06 + | > step_time: 0.1647 (0.24520403943351798) + | > loader_time: 0.0113 (0.011695509573092313) + + + --> TIME: 2024-10-20 05:09:24 -- STEP: 59250/69059 -- GLOBAL_STEP: 59250 + | > loss_text_ce: 0.03681645914912224 (0.03963345025196811) + | > loss_mel_ce: 3.3992269039154053 (3.350966626352384) + | > loss: 0.8590108156204224 (0.8476500191185552) + | > current_lr: 5e-06 + | > step_time: 0.3934 (0.2452134184012433) + | > loader_time: 0.0129 (0.01169546187678468) + + + --> TIME: 2024-10-20 05:09:42 -- STEP: 59300/69059 -- GLOBAL_STEP: 59300 + | > loss_text_ce: 0.030732685700058937 (0.039627872441202215) + | > loss_mel_ce: 3.581880569458008 (3.350825503081521) + | > loss: 0.9031533002853394 (0.8476133438487486) + | > current_lr: 5e-06 + | > step_time: 0.5152 (0.24520814809590125) + | > loader_time: 0.0145 (0.011695395159279094) + + + --> TIME: 2024-10-20 05:10:01 -- STEP: 59350/69059 -- GLOBAL_STEP: 59350 + | > loss_text_ce: 0.034275248646736145 (0.039622340595761876) + | > loss_mel_ce: 2.99051833152771 (3.350663160305603) + | > loss: 0.7561984062194824 (0.8475713751932712) + | > current_lr: 5e-06 + | > step_time: 0.2839 (0.2452064256941258) + | > loader_time: 0.0118 (0.011695202893335055) + + + --> TIME: 2024-10-20 05:10:20 -- STEP: 59400/69059 -- GLOBAL_STEP: 59400 + | > loss_text_ce: 0.03445848822593689 (0.03961674814281767) + | > loss_mel_ce: 3.0226168632507324 (3.350459746186566) + | > loss: 0.7642688155174255 (0.8475191235492132) + | > current_lr: 5e-06 + | > step_time: 0.2074 (0.24520733313929755) + | > loader_time: 0.0114 (0.01169557035571395) + + + --> TIME: 2024-10-20 05:10:39 -- STEP: 59450/69059 -- GLOBAL_STEP: 59450 + | > loss_text_ce: 0.03275676816701889 (0.03961127870195223) + | > loss_mel_ce: 2.8604817390441895 (3.3502969419344475) + | > loss: 0.7233096361160278 (0.8474770551263241) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.24521221204201046) + | > loader_time: 0.0106 (0.011695511311016914) + + + --> TIME: 2024-10-20 05:10:57 -- STEP: 59500/69059 -- GLOBAL_STEP: 59500 + | > loss_text_ce: 0.031346600502729416 (0.039605787544909905) + | > loss_mel_ce: 2.9297714233398438 (3.350153979786289) + | > loss: 0.740279495716095 (0.8474399417989401) + | > current_lr: 5e-06 + | > step_time: 0.1521 (0.24521428022865488) + | > loader_time: 0.011 (0.011695481648966026) + + + --> TIME: 2024-10-20 05:11:17 -- STEP: 59550/69059 -- GLOBAL_STEP: 59550 + | > loss_text_ce: 0.03223780542612076 (0.039600276963974036) + | > loss_mel_ce: 3.4694597721099854 (3.3499848960048357) + | > loss: 0.8754243850708008 (0.8473962932069403) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.24522649469543764) + | > loader_time: 0.0103 (0.011695796664675958) + + + --> TIME: 2024-10-20 05:11:35 -- STEP: 59600/69059 -- GLOBAL_STEP: 59600 + | > loss_text_ce: 0.03233977407217026 (0.039594425682240496) + | > loss_mel_ce: 3.1164302825927734 (3.3497808058789764) + | > loss: 0.7871925234794617 (0.8473438078565093) + | > current_lr: 5e-06 + | > step_time: 0.1813 (0.24521422288561828) + | > loader_time: 0.0109 (0.011695516445492719) + + + --> TIME: 2024-10-20 05:11:53 -- STEP: 59650/69059 -- GLOBAL_STEP: 59650 + | > loss_text_ce: 0.029850462451577187 (0.03958885524400023) + | > loss_mel_ce: 2.9465925693511963 (3.3496071938077296) + | > loss: 0.7441107630729675 (0.8472990122315077) + | > current_lr: 5e-06 + | > step_time: 0.2088 (0.24521613929617436) + | > loader_time: 0.0109 (0.011695263827436608) + + + --> TIME: 2024-10-20 05:12:12 -- STEP: 59700/69059 -- GLOBAL_STEP: 59700 + | > loss_text_ce: 0.030761556699872017 (0.03958367160020638) + | > loss_mel_ce: 3.0216174125671387 (3.349420340760055) + | > loss: 0.7630947232246399 (0.8472510030581132) + | > current_lr: 5e-06 + | > step_time: 0.1807 (0.24521833272036198) + | > loader_time: 0.011 (0.011695049539882315) + + + --> TIME: 2024-10-20 05:12:30 -- STEP: 59750/69059 -- GLOBAL_STEP: 59750 + | > loss_text_ce: 0.03262362256646156 (0.03957798143625834) + | > loss_mel_ce: 3.2210640907287598 (3.3491927682804614) + | > loss: 0.813421905040741 (0.8471926873937319) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.24520471104418384) + | > loader_time: 0.0107 (0.011694572871698975) + + + --> TIME: 2024-10-20 05:12:51 -- STEP: 59800/69059 -- GLOBAL_STEP: 59800 + | > loss_text_ce: 0.03509524092078209 (0.039572809896802036) + | > loss_mel_ce: 3.3715643882751465 (3.3490469345878853) + | > loss: 0.8516649007797241 (0.8471549360867734) + | > current_lr: 5e-06 + | > step_time: 0.3937 (0.2452352871743333) + | > loader_time: 0.0148 (0.011694909964118096) + + + --> TIME: 2024-10-20 05:13:08 -- STEP: 59850/69059 -- GLOBAL_STEP: 59850 + | > loss_text_ce: 0.03421550616621971 (0.03956668849975295) + | > loss_mel_ce: 3.283071279525757 (3.348825470861422) + | > loss: 0.8293216824531555 (0.8470980398039478) + | > current_lr: 5e-06 + | > step_time: 0.1944 (0.2452143687054627) + | > loader_time: 0.0105 (0.011694334941599483) + + + --> TIME: 2024-10-20 05:13:27 -- STEP: 59900/69059 -- GLOBAL_STEP: 59900 + | > loss_text_ce: 0.03155016154050827 (0.039561436784038076) + | > loss_mel_ce: 2.963019609451294 (3.348645842127077) + | > loss: 0.7486424446105957 (0.8470518196912766) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.2452215138222021) + | > loader_time: 0.0106 (0.01169435171133686) + + + --> TIME: 2024-10-20 05:13:45 -- STEP: 59950/69059 -- GLOBAL_STEP: 59950 + | > loss_text_ce: 0.032230816781520844 (0.03955563030127167) + | > loss_mel_ce: 2.9998786449432373 (3.348458578743654) + | > loss: 0.7580273747444153 (0.8470035522276413) + | > current_lr: 5e-06 + | > step_time: 0.1962 (0.24522122890179707) + | > loader_time: 0.0109 (0.01169401479423591) + + + --> TIME: 2024-10-20 05:14:04 -- STEP: 60000/69059 -- GLOBAL_STEP: 60000 + | > loss_text_ce: 0.03320193290710449 (0.03955017937567657) + | > loss_mel_ce: 3.332228899002075 (3.348256971967206) + | > loss: 0.8413577079772949 (0.8469517878015843) + | > current_lr: 5e-06 + | > step_time: 0.3195 (0.24522379415035248) + | > loader_time: 0.0132 (0.011693793372313162) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_60000.pth + + --> TIME: 2024-10-20 05:14:39 -- STEP: 60050/69059 -- GLOBAL_STEP: 60050 + | > loss_text_ce: 0.034605540335178375 (0.03954539995004594) + | > loss_mel_ce: 2.9903147220611572 (3.348112236351678) + | > loss: 0.7562300562858582 (0.8469144090367403) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.24525209895379338) + | > loader_time: 0.0111 (0.011694028903602244) + + + --> TIME: 2024-10-20 05:14:57 -- STEP: 60100/69059 -- GLOBAL_STEP: 60100 + | > loss_text_ce: 0.03229038789868355 (0.0395399112509754) + | > loss_mel_ce: 2.930180549621582 (3.3479297063275157) + | > loss: 0.7406177520751953 (0.846867404354988) + | > current_lr: 5e-06 + | > step_time: 0.1578 (0.24524932634413937) + | > loader_time: 0.0111 (0.011693851749432842) + + + --> TIME: 2024-10-20 05:15:15 -- STEP: 60150/69059 -- GLOBAL_STEP: 60150 + | > loss_text_ce: 0.030639950186014175 (0.039534301236457174) + | > loss_mel_ce: 2.905827045440674 (3.3477894250848994) + | > loss: 0.7341167330741882 (0.84683093153826) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.24524816247094997) + | > loader_time: 0.0107 (0.01169391087859446) + + + --> TIME: 2024-10-20 05:15:34 -- STEP: 60200/69059 -- GLOBAL_STEP: 60200 + | > loss_text_ce: 0.03296775370836258 (0.039528913075982594) + | > loss_mel_ce: 3.4441676139831543 (3.3476123677377143) + | > loss: 0.8692838549613953 (0.8467853201633679) + | > current_lr: 5e-06 + | > step_time: 0.1513 (0.24524786894503622) + | > loader_time: 0.0111 (0.011694018995246995) + + + --> TIME: 2024-10-20 05:15:53 -- STEP: 60250/69059 -- GLOBAL_STEP: 60250 + | > loss_text_ce: 0.035208482295274734 (0.03952350818121533) + | > loss_mel_ce: 3.5656580924987793 (3.3474540643058726) + | > loss: 0.9002166390419006 (0.8467443930825759) + | > current_lr: 5e-06 + | > step_time: 0.2584 (0.2452495844393845) + | > loader_time: 0.0111 (0.01169405180586817) + + + --> TIME: 2024-10-20 05:16:11 -- STEP: 60300/69059 -- GLOBAL_STEP: 60300 + | > loss_text_ce: 0.03369362652301788 (0.03951790682041651) + | > loss_mel_ce: 3.039522886276245 (3.3472367670603114) + | > loss: 0.7683041095733643 (0.8466886684283695) + | > current_lr: 5e-06 + | > step_time: 0.1561 (0.24524550790612773) + | > loader_time: 0.0101 (0.011693880688491726) + + + --> TIME: 2024-10-20 05:16:31 -- STEP: 60350/69059 -- GLOBAL_STEP: 60350 + | > loss_text_ce: 0.03418348729610443 (0.039512582182291145) + | > loss_mel_ce: 3.017449378967285 (3.347089850201312) + | > loss: 0.7629082202911377 (0.8466506080560289) + | > current_lr: 5e-06 + | > step_time: 0.2124 (0.24526537746660157) + | > loader_time: 0.0109 (0.011694221804728643) + + + --> TIME: 2024-10-20 05:16:50 -- STEP: 60400/69059 -- GLOBAL_STEP: 60400 + | > loss_text_ce: 0.03382716327905655 (0.03950683261076594) + | > loss_mel_ce: 3.095576047897339 (3.3468979897759463) + | > loss: 0.7823507785797119 (0.846601205558967) + | > current_lr: 5e-06 + | > step_time: 0.1981 (0.24526865308253182) + | > loader_time: 0.011 (0.01169408552694003) + + + --> TIME: 2024-10-20 05:17:09 -- STEP: 60450/69059 -- GLOBAL_STEP: 60450 + | > loss_text_ce: 0.033655982464551926 (0.03950160550851731) + | > loss_mel_ce: 2.9930996894836426 (3.3466951956405686) + | > loss: 0.7566888928413391 (0.8465492002504543) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.2452701396839593) + | > loader_time: 0.0111 (0.011694077388425797) + + + --> TIME: 2024-10-20 05:17:26 -- STEP: 60500/69059 -- GLOBAL_STEP: 60500 + | > loss_text_ce: 0.0339558869600296 (0.039496358357307845) + | > loss_mel_ce: 3.1840977668762207 (3.3465388997408825) + | > loss: 0.8045133948326111 (0.8465088144895466) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24525845548929262) + | > loader_time: 0.0108 (0.011694031628695383) + + + --> TIME: 2024-10-20 05:17:44 -- STEP: 60550/69059 -- GLOBAL_STEP: 60550 + | > loss_text_ce: 0.033443305641412735 (0.03949115782585424) + | > loss_mel_ce: 3.083036422729492 (3.346389681137253) + | > loss: 0.7791199088096619 (0.8464702097095809) + | > current_lr: 5e-06 + | > step_time: 0.2106 (0.24524586916166727) + | > loader_time: 0.0111 (0.01169380396678365) + + + --> TIME: 2024-10-20 05:18:02 -- STEP: 60600/69059 -- GLOBAL_STEP: 60600 + | > loss_text_ce: 0.030966490507125854 (0.03948579166702288) + | > loss_mel_ce: 3.0425667762756348 (3.346205880563237) + | > loss: 0.7683833241462708 (0.8464229180267943) + | > current_lr: 5e-06 + | > step_time: 0.1523 (0.2452404263861502) + | > loader_time: 0.0102 (0.011693494843177651) + + + --> TIME: 2024-10-20 05:18:19 -- STEP: 60650/69059 -- GLOBAL_STEP: 60650 + | > loss_text_ce: 0.029903016984462738 (0.03948037540412599) + | > loss_mel_ce: 3.160480499267578 (3.3459830569945628) + | > loss: 0.7975958585739136 (0.8463658580687656) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24522958492112376) + | > loader_time: 0.0109 (0.011693176440925296) + + + --> TIME: 2024-10-20 05:18:38 -- STEP: 60700/69059 -- GLOBAL_STEP: 60700 + | > loss_text_ce: 0.03504284471273422 (0.039475048646588115) + | > loss_mel_ce: 3.1896321773529053 (3.3457969640506997) + | > loss: 0.8061687350273132 (0.8463180031444535) + | > current_lr: 5e-06 + | > step_time: 0.4468 (0.245228617682292) + | > loader_time: 0.0148 (0.011693122999672031) + + + --> TIME: 2024-10-20 05:18:57 -- STEP: 60750/69059 -- GLOBAL_STEP: 60750 + | > loss_text_ce: 0.035291481763124466 (0.039469818411325484) + | > loss_mel_ce: 3.1581032276153564 (3.3456314558099676) + | > loss: 0.7983486652374268 (0.8462753185254563) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.24523056485132916) + | > loader_time: 0.0104 (0.011693162435366765) + + + --> TIME: 2024-10-20 05:19:16 -- STEP: 60800/69059 -- GLOBAL_STEP: 60800 + | > loss_text_ce: 0.035076770931482315 (0.039464685783451435) + | > loss_mel_ce: 3.1313726902008057 (3.34543717072589) + | > loss: 0.7916123867034912 (0.8462254640969799) + | > current_lr: 5e-06 + | > step_time: 0.2044 (0.24523925145597833) + | > loader_time: 0.0105 (0.011692873082662866) + + + --> TIME: 2024-10-20 05:19:34 -- STEP: 60850/69059 -- GLOBAL_STEP: 60850 + | > loss_text_ce: 0.03394141048192978 (0.039458983875933065) + | > loss_mel_ce: 2.957594633102417 (3.3452654861443785) + | > loss: 0.7478840351104736 (0.8461811174764539) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.245239567905542) + | > loader_time: 0.011 (0.011692672696579826) + + + --> TIME: 2024-10-20 05:19:53 -- STEP: 60900/69059 -- GLOBAL_STEP: 60900 + | > loss_text_ce: 0.031095629557967186 (0.039453333481191524) + | > loss_mel_ce: 3.0092341899871826 (3.3451070832736245) + | > loss: 0.760082483291626 (0.8461401041583677) + | > current_lr: 5e-06 + | > step_time: 0.1569 (0.24523398072065783) + | > loader_time: 0.0101 (0.011692439765961477) + + + --> TIME: 2024-10-20 05:20:11 -- STEP: 60950/69059 -- GLOBAL_STEP: 60950 + | > loss_text_ce: 0.03471728786826134 (0.03944844055095666) + | > loss_mel_ce: 3.266627550125122 (3.3449674637030005) + | > loss: 0.825336217880249 (0.846103976031031) + | > current_lr: 5e-06 + | > step_time: 0.212 (0.24523058292413952) + | > loader_time: 0.0109 (0.011692159745026243) + + + --> TIME: 2024-10-20 05:20:30 -- STEP: 61000/69059 -- GLOBAL_STEP: 61000 + | > loss_text_ce: 0.03448862209916115 (0.03944343085547281) + | > loss_mel_ce: 3.198669195175171 (3.34479415104036) + | > loss: 0.8082894682884216 (0.8460593954422442) + | > current_lr: 5e-06 + | > step_time: 0.3129 (0.2452386507792551) + | > loader_time: 0.012 (0.01169207298951069) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_61000.pth + + --> TIME: 2024-10-20 05:21:06 -- STEP: 61050/69059 -- GLOBAL_STEP: 61050 + | > loss_text_ce: 0.032712437212467194 (0.039438083565394536) + | > loss_mel_ce: 2.9585208892822266 (3.344644783716711) + | > loss: 0.7478083372116089 (0.846020716790974) + | > current_lr: 5e-06 + | > step_time: 0.2053 (0.24527006073138638) + | > loader_time: 0.0117 (0.011692463471008833) + + + --> TIME: 2024-10-20 05:21:24 -- STEP: 61100/69059 -- GLOBAL_STEP: 61100 + | > loss_text_ce: 0.032394032925367355 (0.03943258707608169) + | > loss_mel_ce: 3.089447021484375 (3.3445010771142534) + | > loss: 0.7804602384567261 (0.8459834160176517) + | > current_lr: 5e-06 + | > step_time: 0.1647 (0.2452641409118515) + | > loader_time: 0.0108 (0.011694335679961091) + + + --> TIME: 2024-10-20 05:21:43 -- STEP: 61150/69059 -- GLOBAL_STEP: 61150 + | > loss_text_ce: 0.033995743840932846 (0.03942746964746997) + | > loss_mel_ce: 3.134753465652466 (3.344320351954654) + | > loss: 0.7921872735023499 (0.8459369553714535) + | > current_lr: 5e-06 + | > step_time: 0.2227 (0.24526538508891668) + | > loader_time: 0.0117 (0.011694350106152744) + + + --> TIME: 2024-10-20 05:22:02 -- STEP: 61200/69059 -- GLOBAL_STEP: 61200 + | > loss_text_ce: 0.0359669104218483 (0.0394223220125323) + | > loss_mel_ce: 3.15783953666687 (3.344167694036463) + | > loss: 0.7984516024589539 (0.8458975039851049) + | > current_lr: 5e-06 + | > step_time: 0.3128 (0.24527460686521593) + | > loader_time: 0.0126 (0.011694316462753627) + + + --> TIME: 2024-10-20 05:22:20 -- STEP: 61250/69059 -- GLOBAL_STEP: 61250 + | > loss_text_ce: 0.03142373636364937 (0.03941653218190233) + | > loss_mel_ce: 2.684457778930664 (3.343981307193197) + | > loss: 0.6789703965187073 (0.8458494598135663) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.2452687759477265) + | > loader_time: 0.0106 (0.011693967029026558) + + + --> TIME: 2024-10-20 05:22:39 -- STEP: 61300/69059 -- GLOBAL_STEP: 61300 + | > loss_text_ce: 0.03278685361146927 (0.039411460741332956) + | > loss_mel_ce: 2.9258198738098145 (3.3437878322990113) + | > loss: 0.7396516799926758 (0.8457998232293097) + | > current_lr: 5e-06 + | > step_time: 0.1633 (0.24526952938387872) + | > loader_time: 0.0105 (0.011693930077513882) + + + --> TIME: 2024-10-20 05:22:57 -- STEP: 61350/69059 -- GLOBAL_STEP: 61350 + | > loss_text_ce: 0.03310081735253334 (0.03940603855918338) + | > loss_mel_ce: 3.222822427749634 (3.3436047258120443) + | > loss: 0.8139808177947998 (0.8457526910635358) + | > current_lr: 5e-06 + | > step_time: 0.2314 (0.24526705773360571) + | > loader_time: 0.0109 (0.011693681974962834) + + + --> TIME: 2024-10-20 05:23:15 -- STEP: 61400/69059 -- GLOBAL_STEP: 61400 + | > loss_text_ce: 0.0325683131814003 (0.03940080575377963) + | > loss_mel_ce: 3.1769466400146484 (3.343414109630757) + | > loss: 0.8023787140846252 (0.8457037288183505) + | > current_lr: 5e-06 + | > step_time: 0.1577 (0.24526159858858934) + | > loader_time: 0.0116 (0.011693448927192793) + + + --> TIME: 2024-10-20 05:23:34 -- STEP: 61450/69059 -- GLOBAL_STEP: 61450 + | > loss_text_ce: 0.03421046584844589 (0.03939562894440261) + | > loss_mel_ce: 3.2561373710632324 (3.343220559338213) + | > loss: 0.8225869536399841 (0.8456540470447268) + | > current_lr: 5e-06 + | > step_time: 0.5312 (0.24526445605291206) + | > loader_time: 0.0147 (0.011695347181464503) + + + --> TIME: 2024-10-20 05:23:52 -- STEP: 61500/69059 -- GLOBAL_STEP: 61500 + | > loss_text_ce: 0.03169602155685425 (0.03939052544509929) + | > loss_mel_ce: 2.8449180126190186 (3.3430217142376333) + | > loss: 0.7191535234451294 (0.845603059894671) + | > current_lr: 5e-06 + | > step_time: 0.1806 (0.24525367608884485) + | > loader_time: 0.0111 (0.011695220392893951) + + + --> TIME: 2024-10-20 05:24:11 -- STEP: 61550/69059 -- GLOBAL_STEP: 61550 + | > loss_text_ce: 0.03500882536172867 (0.03938580701207796) + | > loss_mel_ce: 3.187896728515625 (3.342845350047794) + | > loss: 0.8057264089584351 (0.8455577892360415) + | > current_lr: 5e-06 + | > step_time: 0.2491 (0.24526604246066702) + | > loader_time: 0.011 (0.011695155655824472) + + + --> TIME: 2024-10-20 05:24:30 -- STEP: 61600/69059 -- GLOBAL_STEP: 61600 + | > loss_text_ce: 0.03417782112956047 (0.03938053712569275) + | > loss_mel_ce: 3.3275582790374756 (3.3426998390044345) + | > loss: 0.8404340147972107 (0.8455200940031904) + | > current_lr: 5e-06 + | > step_time: 0.2863 (0.24526478267335272) + | > loader_time: 0.0127 (0.011695051208719012) + + + --> TIME: 2024-10-20 05:24:49 -- STEP: 61650/69059 -- GLOBAL_STEP: 61650 + | > loss_text_ce: 0.03129702806472778 (0.039375454960518856) + | > loss_mel_ce: 3.1182432174682617 (3.342541300227545) + | > loss: 0.7873850464820862 (0.845479188767857) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.24526875798332914) + | > loader_time: 0.0109 (0.011694881910245186) + + + --> TIME: 2024-10-20 05:25:07 -- STEP: 61700/69059 -- GLOBAL_STEP: 61700 + | > loss_text_ce: 0.032985836267471313 (0.039370318683017796) + | > loss_mel_ce: 3.2917871475219727 (3.3423697344758185) + | > loss: 0.8311932682991028 (0.8454350132580908) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.24526345098810798) + | > loader_time: 0.0107 (0.011694519473435987) + + + --> TIME: 2024-10-20 05:25:25 -- STEP: 61750/69059 -- GLOBAL_STEP: 61750 + | > loss_text_ce: 0.03344728425145149 (0.03936521303605407) + | > loss_mel_ce: 3.4509482383728027 (3.3422074689054155) + | > loss: 0.8710988759994507 (0.8453931704542422) + | > current_lr: 5e-06 + | > step_time: 0.2234 (0.24526276614694942) + | > loader_time: 0.0116 (0.0116943093620331) + + + --> TIME: 2024-10-20 05:25:43 -- STEP: 61800/69059 -- GLOBAL_STEP: 61800 + | > loss_text_ce: 0.030042123049497604 (0.03935982177886932) + | > loss_mel_ce: 3.3994367122650146 (3.34205420615988) + | > loss: 0.8573697209358215 (0.845353506954936) + | > current_lr: 5e-06 + | > step_time: 0.1742 (0.24525867249587593) + | > loader_time: 0.0104 (0.011693980061120567) + + + --> TIME: 2024-10-20 05:26:01 -- STEP: 61850/69059 -- GLOBAL_STEP: 61850 + | > loss_text_ce: 0.031759150326251984 (0.0393545186189552) + | > loss_mel_ce: 2.7840588092803955 (3.341848933829723) + | > loss: 0.7039545178413391 (0.8453008630811549) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.24524317408340443) + | > loader_time: 0.0106 (0.011693559745288976) + + + --> TIME: 2024-10-20 05:26:18 -- STEP: 61900/69059 -- GLOBAL_STEP: 61900 + | > loss_text_ce: 0.03034610114991665 (0.039349334018909304) + | > loss_mel_ce: 3.242812395095825 (3.3416675649667456) + | > loss: 0.8182896375656128 (0.845254224714055) + | > current_lr: 5e-06 + | > step_time: 0.1819 (0.24522897533915924) + | > loader_time: 0.011 (0.01169310692631561) + + + --> TIME: 2024-10-20 05:26:36 -- STEP: 61950/69059 -- GLOBAL_STEP: 61950 + | > loss_text_ce: 0.030430736020207405 (0.03934371959433045) + | > loss_mel_ce: 2.8803231716156006 (3.3415037891343156) + | > loss: 0.7276884913444519 (0.8452118771474536) + | > current_lr: 5e-06 + | > step_time: 0.2017 (0.2452152327244276) + | > loader_time: 0.011 (0.011692793675254083) + + + --> TIME: 2024-10-20 05:26:54 -- STEP: 62000/69059 -- GLOBAL_STEP: 62000 + | > loss_text_ce: 0.03194520249962807 (0.0393388757407961) + | > loss_mel_ce: 3.7644011974334717 (3.3413381825723816) + | > loss: 0.949086606502533 (0.8451692645444031) + | > current_lr: 5e-06 + | > step_time: 0.2753 (0.24520939695835114) + | > loader_time: 0.0132 (0.011692537646139802) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_62000.pth + + --> TIME: 2024-10-20 05:27:28 -- STEP: 62050/69059 -- GLOBAL_STEP: 62050 + | > loss_text_ce: 0.03551999107003212 (0.03933367777798851) + | > loss_mel_ce: 3.4333255290985107 (3.3411789781791614) + | > loss: 0.8672114014625549 (0.8451281639551759) + | > current_lr: 5e-06 + | > step_time: 0.4216 (0.24523022759642743) + | > loader_time: 0.0134 (0.011692925364043212) + + + --> TIME: 2024-10-20 05:27:47 -- STEP: 62100/69059 -- GLOBAL_STEP: 62100 + | > loss_text_ce: 0.03444652631878853 (0.039328377719776125) + | > loss_mel_ce: 2.936145782470703 (3.3410280434443997) + | > loss: 0.7426480650901794 (0.8450891052534429) + | > current_lr: 5e-06 + | > step_time: 0.207 (0.24523953554902866) + | > loader_time: 0.0113 (0.011693238989358536) + + + --> TIME: 2024-10-20 05:28:07 -- STEP: 62150/69059 -- GLOBAL_STEP: 62150 + | > loss_text_ce: 0.03043769858777523 (0.03932327520359959) + | > loss_mel_ce: 2.987440347671509 (3.3408727792339232) + | > loss: 0.7544695138931274 (0.845049013572649) + | > current_lr: 5e-06 + | > step_time: 0.179 (0.24525135983599833) + | > loader_time: 0.0108 (0.01169358073178548) + + + --> TIME: 2024-10-20 05:28:24 -- STEP: 62200/69059 -- GLOBAL_STEP: 62200 + | > loss_text_ce: 0.03144057095050812 (0.03931795512499702) + | > loss_mel_ce: 3.4596216678619385 (3.340685817957677) + | > loss: 0.8727655410766602 (0.8450009432310467) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.24523954470441273) + | > loader_time: 0.0111 (0.011693283177648691) + + + --> TIME: 2024-10-20 05:28:43 -- STEP: 62250/69059 -- GLOBAL_STEP: 62250 + | > loss_text_ce: 0.030265768989920616 (0.03931305933920208) + | > loss_mel_ce: 3.0982885360717773 (3.3405593187607776) + | > loss: 0.7821385860443115 (0.8449680944848738) + | > current_lr: 5e-06 + | > step_time: 0.1773 (0.24524572129996425) + | > loader_time: 0.0115 (0.011693481142741113) + + + --> TIME: 2024-10-20 05:29:00 -- STEP: 62300/69059 -- GLOBAL_STEP: 62300 + | > loss_text_ce: 0.03276115655899048 (0.03930765583109584) + | > loss_mel_ce: 2.8645951747894287 (3.340352345362507) + | > loss: 0.7243390679359436 (0.8449150002576764) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.24522535704494863) + | > loader_time: 0.0105 (0.011693116478131614) + + + --> TIME: 2024-10-20 05:29:19 -- STEP: 62350/69059 -- GLOBAL_STEP: 62350 + | > loss_text_ce: 0.032742612063884735 (0.03930270867196627) + | > loss_mel_ce: 2.9900755882263184 (3.3402024295756267) + | > loss: 0.7557045221328735 (0.8448762845190035) + | > current_lr: 5e-06 + | > step_time: 0.2058 (0.24523258963678968) + | > loader_time: 0.0106 (0.01169301170679502) + + + --> TIME: 2024-10-20 05:29:38 -- STEP: 62400/69059 -- GLOBAL_STEP: 62400 + | > loss_text_ce: 0.03196970745921135 (0.039297872464327296) + | > loss_mel_ce: 3.24392032623291 (3.3399924759184563) + | > loss: 0.8189725279808044 (0.8448225870537459) + | > current_lr: 5e-06 + | > step_time: 0.1556 (0.2452271708731468) + | > loader_time: 0.0102 (0.011692843995033148) + + + --> TIME: 2024-10-20 05:29:56 -- STEP: 62450/69059 -- GLOBAL_STEP: 62450 + | > loss_text_ce: 0.03125620633363724 (0.039292963665929735) + | > loss_mel_ce: 3.197862386703491 (3.339832590189417) + | > loss: 0.807279646396637 (0.844781388421361) + | > current_lr: 5e-06 + | > step_time: 0.2232 (0.24522735404052) + | > loader_time: 0.0111 (0.011692630066883078) + + + --> TIME: 2024-10-20 05:30:15 -- STEP: 62500/69059 -- GLOBAL_STEP: 62500 + | > loss_text_ce: 0.036589596420526505 (0.03928786172893597) + | > loss_mel_ce: 2.986198663711548 (3.339706358013139) + | > loss: 0.7556970715522766 (0.844748554894448) + | > current_lr: 5e-06 + | > step_time: 0.1723 (0.24523450774765015) + | > loader_time: 0.0123 (0.011692502189636212) + + + --> TIME: 2024-10-20 05:30:33 -- STEP: 62550/69059 -- GLOBAL_STEP: 62550 + | > loss_text_ce: 0.029007289558649063 (0.039282445165035644) + | > loss_mel_ce: 2.9843547344207764 (3.339532546124189) + | > loss: 0.753340482711792 (0.8447037477821094) + | > current_lr: 5e-06 + | > step_time: 0.1715 (0.2452271273915621) + | > loader_time: 0.0102 (0.011692319800623106) + + + --> TIME: 2024-10-20 05:30:52 -- STEP: 62600/69059 -- GLOBAL_STEP: 62600 + | > loss_text_ce: 0.034242745488882065 (0.03927720345839359) + | > loss_mel_ce: 3.4074974060058594 (3.339363075430006) + | > loss: 0.8604350090026855 (0.8446600696834901) + | > current_lr: 5e-06 + | > step_time: 0.2449 (0.24523011743451079) + | > loader_time: 0.012 (0.011692333213818322) + + + --> TIME: 2024-10-20 05:31:10 -- STEP: 62650/69059 -- GLOBAL_STEP: 62650 + | > loss_text_ce: 0.029400929808616638 (0.03927217813899898) + | > loss_mel_ce: 3.0461089611053467 (3.339165999118733) + | > loss: 0.7688774466514587 (0.8446095442733863) + | > current_lr: 5e-06 + | > step_time: 0.234 (0.24522094852526666) + | > loader_time: 0.0113 (0.01169202150008435) + + + --> TIME: 2024-10-20 05:31:28 -- STEP: 62700/69059 -- GLOBAL_STEP: 62700 + | > loss_text_ce: 0.03192541003227234 (0.0392673476524587) + | > loss_mel_ce: 3.044128656387329 (3.339001097123955) + | > loss: 0.769013524055481 (0.8445671111544931) + | > current_lr: 5e-06 + | > step_time: 0.1725 (0.24521728330061576) + | > loader_time: 0.0105 (0.011691630951335347) + + + --> TIME: 2024-10-20 05:31:46 -- STEP: 62750/69059 -- GLOBAL_STEP: 62750 + | > loss_text_ce: 0.031491320580244064 (0.03926233447892211) + | > loss_mel_ce: 3.2718687057495117 (3.3388138149838857) + | > loss: 0.8258399963378906 (0.8445190373274442) + | > current_lr: 5e-06 + | > step_time: 0.2316 (0.24521492759734986) + | > loader_time: 0.0115 (0.011691526538347326) + + + --> TIME: 2024-10-20 05:32:03 -- STEP: 62800/69059 -- GLOBAL_STEP: 62800 + | > loss_text_ce: 0.03211183845996857 (0.03925746876767132) + | > loss_mel_ce: 2.9721713066101074 (3.338590167438135) + | > loss: 0.7510707974433899 (0.8444619090114817) + | > current_lr: 5e-06 + | > step_time: 0.1519 (0.24519555254726652) + | > loader_time: 0.0105 (0.011691055635737748) + + + --> TIME: 2024-10-20 05:32:22 -- STEP: 62850/69059 -- GLOBAL_STEP: 62850 + | > loss_text_ce: 0.03307293355464935 (0.03925233943116647) + | > loss_mel_ce: 3.5457897186279297 (3.3384030873109594) + | > loss: 0.8947156667709351 (0.8444138566425333) + | > current_lr: 5e-06 + | > step_time: 0.4861 (0.24519635238435028) + | > loader_time: 0.0148 (0.011690841976763601) + + + --> TIME: 2024-10-20 05:32:40 -- STEP: 62900/69059 -- GLOBAL_STEP: 62900 + | > loss_text_ce: 0.03591689094901085 (0.03924710557536903) + | > loss_mel_ce: 3.067197799682617 (3.3382193543573493) + | > loss: 0.7757786512374878 (0.8443666149417629) + | > current_lr: 5e-06 + | > step_time: 0.3127 (0.24519042423305906) + | > loader_time: 0.012 (0.011690548269851012) + + + --> TIME: 2024-10-20 05:32:58 -- STEP: 62950/69059 -- GLOBAL_STEP: 62950 + | > loss_text_ce: 0.031672678887844086 (0.039241934362635726) + | > loss_mel_ce: 3.3146557807922363 (3.3380718816093506) + | > loss: 0.8365821242332458 (0.8443284539526855) + | > current_lr: 5e-06 + | > step_time: 0.1722 (0.24518944130140793) + | > loader_time: 0.0105 (0.011690331857482795) + + + --> TIME: 2024-10-20 05:33:18 -- STEP: 63000/69059 -- GLOBAL_STEP: 63000 + | > loss_text_ce: 0.033796750009059906 (0.039236919933929575) + | > loss_mel_ce: 3.1360886096954346 (3.33791361739899) + | > loss: 0.7924713492393494 (0.8442876342951312) + | > current_lr: 5e-06 + | > step_time: 0.1969 (0.2452064473326244) + | > loader_time: 0.0113 (0.011690386230983413) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_63000.pth + + --> TIME: 2024-10-20 05:33:52 -- STEP: 63050/69059 -- GLOBAL_STEP: 63050 + | > loss_text_ce: 0.029781384393572807 (0.03923208290759744) + | > loss_mel_ce: 2.8965868949890137 (3.3377344995562046) + | > loss: 0.731592059135437 (0.8442416455800534) + | > current_lr: 5e-06 + | > step_time: 0.1936 (0.24521156145030784) + | > loader_time: 0.0105 (0.011690382454529517) + + + --> TIME: 2024-10-20 05:34:10 -- STEP: 63100/69059 -- GLOBAL_STEP: 63100 + | > loss_text_ce: 0.030694998800754547 (0.03922668060075251) + | > loss_mel_ce: 3.1530849933624268 (3.3375664286870257) + | > loss: 0.7959449887275696 (0.8441982772866073) + | > current_lr: 5e-06 + | > step_time: 0.199 (0.24520455475654543) + | > loader_time: 0.0112 (0.01169018863687045) + + + --> TIME: 2024-10-20 05:34:27 -- STEP: 63150/69059 -- GLOBAL_STEP: 63150 + | > loss_text_ce: 0.03313977271318436 (0.039221617887540895) + | > loss_mel_ce: 3.0851969718933105 (3.337408647148348) + | > loss: 0.7795841693878174 (0.8441575662238399) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.2451895060769547) + | > loader_time: 0.0112 (0.011690118182583245) + + + --> TIME: 2024-10-20 05:34:44 -- STEP: 63200/69059 -- GLOBAL_STEP: 63200 + | > loss_text_ce: 0.029833471402525902 (0.039216531325585065) + | > loss_mel_ce: 3.237337827682495 (3.3372570070888283) + | > loss: 0.8167928457260132 (0.8441183845670547) + | > current_lr: 5e-06 + | > step_time: 0.1581 (0.24517553636544867) + | > loader_time: 0.0117 (0.011690194380434235) + + + --> TIME: 2024-10-20 05:35:02 -- STEP: 63250/69059 -- GLOBAL_STEP: 63250 + | > loss_text_ce: 0.03479091450572014 (0.03921151832155396) + | > loss_mel_ce: 3.697920560836792 (3.337062318605854) + | > loss: 0.9331778883934021 (0.844068459196996) + | > current_lr: 5e-06 + | > step_time: 0.2345 (0.24516113749040444) + | > loader_time: 0.0119 (0.011689989440525926) + + + --> TIME: 2024-10-20 05:35:18 -- STEP: 63300/69059 -- GLOBAL_STEP: 63300 + | > loss_text_ce: 0.032062701880931854 (0.039206399248903644) + | > loss_mel_ce: 3.180271863937378 (3.3369084810306537) + | > loss: 0.8030836582183838 (0.8440287200346196) + | > current_lr: 5e-06 + | > step_time: 0.1966 (0.24514004165715705) + | > loader_time: 0.0111 (0.01168973042125007) + + + --> TIME: 2024-10-20 05:35:37 -- STEP: 63350/69059 -- GLOBAL_STEP: 63350 + | > loss_text_ce: 0.030093004927039146 (0.039201295693017134) + | > loss_mel_ce: 3.2226171493530273 (3.336746088613543) + | > loss: 0.8131775259971619 (0.8439868460392149) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.24514557116974406) + | > loader_time: 0.0105 (0.01168979253746338) + + + --> TIME: 2024-10-20 05:35:56 -- STEP: 63400/69059 -- GLOBAL_STEP: 63400 + | > loss_text_ce: 0.03169567137956619 (0.03919624872731839) + | > loss_mel_ce: 3.0706238746643066 (3.3365552003714427) + | > loss: 0.7755798697471619 (0.8439378622371698) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.245144999798164) + | > loader_time: 0.0106 (0.011689999697712299) + + + --> TIME: 2024-10-20 05:36:13 -- STEP: 63450/69059 -- GLOBAL_STEP: 63450 + | > loss_text_ce: 0.035386230796575546 (0.039191004364736874) + | > loss_mel_ce: 2.9710135459899902 (3.3363741735814454) + | > loss: 0.7515999674797058 (0.8438912944468677) + | > current_lr: 5e-06 + | > step_time: 0.2762 (0.2451362295391242) + | > loader_time: 0.0123 (0.011689819534337812) + + + --> TIME: 2024-10-20 05:36:30 -- STEP: 63500/69059 -- GLOBAL_STEP: 63500 + | > loss_text_ce: 0.031097689643502235 (0.039185407330288906) + | > loss_mel_ce: 3.637019157409668 (3.3361869621915003) + | > loss: 0.9170292019844055 (0.8438430923430007) + | > current_lr: 5e-06 + | > step_time: 0.3459 (0.24511762144058707) + | > loader_time: 0.0128 (0.011689538302384006) + + + --> TIME: 2024-10-20 05:36:48 -- STEP: 63550/69059 -- GLOBAL_STEP: 63550 + | > loss_text_ce: 0.03275979310274124 (0.03918022493801725) + | > loss_mel_ce: 3.0414366722106934 (3.3360285386881476) + | > loss: 0.7685491442680359 (0.8438021908689921) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.24511023365729068) + | > loader_time: 0.011 (0.011689467220021251) + + + --> TIME: 2024-10-20 05:37:07 -- STEP: 63600/69059 -- GLOBAL_STEP: 63600 + | > loss_text_ce: 0.03455901890993118 (0.03917552498221914) + | > loss_mel_ce: 2.9591336250305176 (3.3358463320679497) + | > loss: 0.748423159122467 (0.8437554642267578) + | > current_lr: 5e-06 + | > step_time: 0.1567 (0.24511023814573227) + | > loader_time: 0.0108 (0.011689251597572404) + + + --> TIME: 2024-10-20 05:37:27 -- STEP: 63650/69059 -- GLOBAL_STEP: 63650 + | > loss_text_ce: 0.03138305991888046 (0.0391705004940535) + | > loss_mel_ce: 3.2242836952209473 (3.3356964156239832) + | > loss: 0.8139166831970215 (0.8437167289944406) + | > current_lr: 5e-06 + | > step_time: 0.1951 (0.2451299733209123) + | > loader_time: 0.0107 (0.011689160284055825) + + + --> TIME: 2024-10-20 05:37:45 -- STEP: 63700/69059 -- GLOBAL_STEP: 63700 + | > loss_text_ce: 0.033477332442998886 (0.03916551578816221) + | > loss_mel_ce: 2.9518473148345947 (3.3355016512923052) + | > loss: 0.7463311553001404 (0.8436667917343668) + | > current_lr: 5e-06 + | > step_time: 0.1994 (0.24512663551926425) + | > loader_time: 0.0154 (0.011688963693202553) + + + --> TIME: 2024-10-20 05:38:05 -- STEP: 63750/69059 -- GLOBAL_STEP: 63750 + | > loss_text_ce: 0.03612768277525902 (0.03916060646518143) + | > loss_mel_ce: 3.1994242668151855 (3.3353265583711376) + | > loss: 0.8088879585266113 (0.8436217911710933) + | > current_lr: 5e-06 + | > step_time: 0.3791 (0.24514367811539595) + | > loader_time: 0.0131 (0.011689039301404748) + + + --> TIME: 2024-10-20 05:38:24 -- STEP: 63800/69059 -- GLOBAL_STEP: 63800 + | > loss_text_ce: 0.030573291704058647 (0.039155641491509824) + | > loss_mel_ce: 3.0716500282287598 (3.335156535612738) + | > loss: 0.7755558490753174 (0.8435780442386965) + | > current_lr: 5e-06 + | > step_time: 0.1483 (0.24514810397333486) + | > loader_time: 0.0102 (0.011688828404794275) + + + --> TIME: 2024-10-20 05:38:42 -- STEP: 63850/69059 -- GLOBAL_STEP: 63850 + | > loss_text_ce: 0.03359589725732803 (0.03915075483843464) + | > loss_mel_ce: 3.0442020893096924 (3.334997082929067) + | > loss: 0.7694494724273682 (0.8435369594048602) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.24513910051613183) + | > loader_time: 0.0116 (0.011688486680260075) + + + --> TIME: 2024-10-20 05:38:59 -- STEP: 63900/69059 -- GLOBAL_STEP: 63900 + | > loss_text_ce: 0.03534039482474327 (0.039145858239043915) + | > loss_mel_ce: 3.611739158630371 (3.3348386661920424) + | > loss: 0.9117698669433594 (0.8434961310667991) + | > current_lr: 5e-06 + | > step_time: 0.3033 (0.24511750082827882) + | > loader_time: 0.0127 (0.011688065678114418) + + + --> TIME: 2024-10-20 05:39:17 -- STEP: 63950/69059 -- GLOBAL_STEP: 63950 + | > loss_text_ce: 0.03482666239142418 (0.03914086799188948) + | > loss_mel_ce: 3.0590546131134033 (3.3346507412013318) + | > loss: 0.7734703421592712 (0.8434479022575979) + | > current_lr: 5e-06 + | > step_time: 0.6057 (0.24510995252697237) + | > loader_time: 0.0151 (0.011687879677952203) + + + --> TIME: 2024-10-20 05:39:35 -- STEP: 64000/69059 -- GLOBAL_STEP: 64000 + | > loss_text_ce: 0.03450657054781914 (0.0391359045839049) + | > loss_mel_ce: 3.39510440826416 (3.3345384055860205) + | > loss: 0.8574027419090271 (0.8434185775006197) + | > current_lr: 5e-06 + | > step_time: 0.207 (0.24511147901043295) + | > loader_time: 0.0118 (0.011687704484909755) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_64000.pth + + --> TIME: 2024-10-20 05:40:08 -- STEP: 64050/69059 -- GLOBAL_STEP: 64050 + | > loss_text_ce: 0.03224123641848564 (0.039130900324984154) + | > loss_mel_ce: 2.9296603202819824 (3.3343382668141417) + | > loss: 0.7404754161834717 (0.843367291745686) + | > current_lr: 5e-06 + | > step_time: 0.1948 (0.2451027455504847) + | > loader_time: 0.0118 (0.0116874596516105) + + + --> TIME: 2024-10-20 05:40:25 -- STEP: 64100/69059 -- GLOBAL_STEP: 64100 + | > loss_text_ce: 0.03326806798577309 (0.03912608287699603) + | > loss_mel_ce: 3.39213490486145 (3.334146970411903) + | > loss: 0.8563507199287415 (0.8433182632858042) + | > current_lr: 5e-06 + | > step_time: 0.3462 (0.24509415819566818) + | > loader_time: 0.0128 (0.011687220634424724) + + + --> TIME: 2024-10-20 05:40:45 -- STEP: 64150/69059 -- GLOBAL_STEP: 64150 + | > loss_text_ce: 0.0301288403570652 (0.0391211839198237) + | > loss_mel_ce: 2.9051427841186523 (3.333988554282558) + | > loss: 0.7338179349899292 (0.8432774345129619) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.24510664506898852) + | > loader_time: 0.0103 (0.011687137918925694) + + + --> TIME: 2024-10-20 05:41:02 -- STEP: 64200/69059 -- GLOBAL_STEP: 64200 + | > loss_text_ce: 0.03410360589623451 (0.03911620824179941) + | > loss_mel_ce: 3.017665147781372 (3.3337848158751555) + | > loss: 0.7629421949386597 (0.8432252559919973) + | > current_lr: 5e-06 + | > step_time: 0.2505 (0.2450880892180208) + | > loader_time: 0.0119 (0.011686844253837116) + + + --> TIME: 2024-10-20 05:41:20 -- STEP: 64250/69059 -- GLOBAL_STEP: 64250 + | > loss_text_ce: 0.03093838319182396 (0.0391110749055087) + | > loss_mel_ce: 3.0047218799591064 (3.3336254605504654) + | > loss: 0.7589150667190552 (0.843184133827455) + | > current_lr: 5e-06 + | > step_time: 0.1946 (0.24507646214080692) + | > loader_time: 0.0108 (0.01168653551242693) + + + --> TIME: 2024-10-20 05:41:38 -- STEP: 64300/69059 -- GLOBAL_STEP: 64300 + | > loss_text_ce: 0.03386533260345459 (0.03910617060278545) + | > loss_mel_ce: 2.7910196781158447 (3.333450456481315) + | > loss: 0.7062212228775024 (0.8431391567306985) + | > current_lr: 5e-06 + | > step_time: 0.1648 (0.2450746665141816) + | > loader_time: 0.0108 (0.011686391081446606) + + + --> TIME: 2024-10-20 05:41:56 -- STEP: 64350/69059 -- GLOBAL_STEP: 64350 + | > loss_text_ce: 0.030425850301980972 (0.03910142201626311) + | > loss_mel_ce: 2.947824001312256 (3.3332879555753205) + | > loss: 0.7445624470710754 (0.8430973443581145) + | > current_lr: 5e-06 + | > step_time: 0.177 (0.24507262240664016) + | > loader_time: 0.0114 (0.01168644520137877) + + + --> TIME: 2024-10-20 05:42:16 -- STEP: 64400/69059 -- GLOBAL_STEP: 64400 + | > loss_text_ce: 0.031090373173356056 (0.03909675469908309) + | > loss_mel_ce: 3.033811092376709 (3.3331578136156668) + | > loss: 0.7662253379821777 (0.8430636420383223) + | > current_lr: 5e-06 + | > step_time: 0.1493 (0.24508078013147627) + | > loader_time: 0.0109 (0.011686406513178553) + + + --> TIME: 2024-10-20 05:42:35 -- STEP: 64450/69059 -- GLOBAL_STEP: 64450 + | > loss_text_ce: 0.031593210995197296 (0.03909180371161531) + | > loss_mel_ce: 3.0111546516418457 (3.332998833611906) + | > loss: 0.760686993598938 (0.843022659289181) + | > current_lr: 5e-06 + | > step_time: 0.2234 (0.24509211789183694) + | > loader_time: 0.0111 (0.011686340676804104) + + + --> TIME: 2024-10-20 05:42:54 -- STEP: 64500/69059 -- GLOBAL_STEP: 64500 + | > loss_text_ce: 0.03288828209042549 (0.03908676635670293) + | > loss_mel_ce: 3.007016897201538 (3.332843587875353) + | > loss: 0.7599762678146362 (0.8429825885175741) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.2450923574358918) + | > loader_time: 0.0109 (0.011686274103416012) + + + --> TIME: 2024-10-20 05:43:10 -- STEP: 64550/69059 -- GLOBAL_STEP: 64550 + | > loss_text_ce: 0.03505302965641022 (0.03908182271220189) + | > loss_mel_ce: 3.264373540878296 (3.332677182190909) + | > loss: 0.8248566389083862 (0.8429397511833132) + | > current_lr: 5e-06 + | > step_time: 0.2231 (0.2450735106235507) + | > loader_time: 0.0103 (0.011685900001171281) + + + --> TIME: 2024-10-20 05:43:28 -- STEP: 64600/69059 -- GLOBAL_STEP: 64600 + | > loss_text_ce: 0.03521957993507385 (0.039077188207309) + | > loss_mel_ce: 3.2841572761535645 (3.3325114456271105) + | > loss: 0.8298442363739014 (0.8428971584127422) + | > current_lr: 5e-06 + | > step_time: 0.1529 (0.24506143923883467) + | > loader_time: 0.0112 (0.01168578435762008) + + + --> TIME: 2024-10-20 05:43:46 -- STEP: 64650/69059 -- GLOBAL_STEP: 64650 + | > loss_text_ce: 0.031683698296546936 (0.0390722587393781) + | > loss_mel_ce: 3.2245333194732666 (3.3323677960717997) + | > loss: 0.8140542507171631 (0.8428600136540427) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.24505520143899823) + | > loader_time: 0.0105 (0.011685561307713845) + + + --> TIME: 2024-10-20 05:44:04 -- STEP: 64700/69059 -- GLOBAL_STEP: 64700 + | > loss_text_ce: 0.032425981014966965 (0.039067174912396786) + | > loss_mel_ce: 2.780813455581665 (3.332200526369397) + | > loss: 0.7033098340034485 (0.8428169252745342) + | > current_lr: 5e-06 + | > step_time: 0.1524 (0.2450464746425104) + | > loader_time: 0.0102 (0.011685272895571235) + + + --> TIME: 2024-10-20 05:44:23 -- STEP: 64750/69059 -- GLOBAL_STEP: 64750 + | > loss_text_ce: 0.032268673181533813 (0.03906252353327685) + | > loss_mel_ce: 3.183889865875244 (3.332056068825432) + | > loss: 0.8040396571159363 (0.8427796480453159) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.2450561720837037) + | > loader_time: 0.0114 (0.011685221392215431) + + + --> TIME: 2024-10-20 05:44:43 -- STEP: 64800/69059 -- GLOBAL_STEP: 64800 + | > loss_text_ce: 0.03571191802620888 (0.03905769942213311) + | > loss_mel_ce: 3.522681474685669 (3.331955071356548) + | > loss: 0.8895983695983887 (0.8427531926527074) + | > current_lr: 5e-06 + | > step_time: 0.2973 (0.24507042836627843) + | > loader_time: 0.0156 (0.011685228152775453) + + + --> TIME: 2024-10-20 05:45:01 -- STEP: 64850/69059 -- GLOBAL_STEP: 64850 + | > loss_text_ce: 0.03303404897451401 (0.03905258200999445) + | > loss_mel_ce: 3.3783981800079346 (3.331811007637181) + | > loss: 0.8528580665588379 (0.8427158973735398) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.24506474767727585) + | > loader_time: 0.0103 (0.011684894558091099) + + + --> TIME: 2024-10-20 05:45:19 -- STEP: 64900/69059 -- GLOBAL_STEP: 64900 + | > loss_text_ce: 0.03209128603339195 (0.03904783970491201) + | > loss_mel_ce: 3.13189435005188 (3.331644521920436) + | > loss: 0.7909964323043823 (0.8426730903693824) + | > current_lr: 5e-06 + | > step_time: 0.2841 (0.24505801362873778) + | > loader_time: 0.0119 (0.011684588743101082) + + + --> TIME: 2024-10-20 05:45:38 -- STEP: 64950/69059 -- GLOBAL_STEP: 64950 + | > loss_text_ce: 0.03127719461917877 (0.0390432329522047) + | > loss_mel_ce: 3.003613233566284 (3.331475980680098) + | > loss: 0.7587226033210754 (0.8426298033732288) + | > current_lr: 5e-06 + | > step_time: 0.1941 (0.2450664070003119) + | > loader_time: 0.011 (0.011684564978824918) + + + --> TIME: 2024-10-20 05:45:58 -- STEP: 65000/69059 -- GLOBAL_STEP: 65000 + | > loss_text_ce: 0.0315052755177021 (0.03903816731348589) + | > loss_mel_ce: 3.067800998687744 (3.331338100330633) + | > loss: 0.7748265862464905 (0.8425940668784662) + | > current_lr: 5e-06 + | > step_time: 0.1652 (0.24507683219909668) + | > loader_time: 0.0117 (0.011684536273662843) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_65000.pth + + --> TIME: 2024-10-20 05:46:33 -- STEP: 65050/69059 -- GLOBAL_STEP: 65050 + | > loss_text_ce: 0.033808860927820206 (0.03903344059876944) + | > loss_mel_ce: 2.9860739707946777 (3.3311581741526393) + | > loss: 0.7549707293510437 (0.842547903656502) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.24507400547514321) + | > loader_time: 0.011 (0.011684377690079926) + + + --> TIME: 2024-10-20 05:46:50 -- STEP: 65100/69059 -- GLOBAL_STEP: 65100 + | > loss_text_ce: 0.03336973488330841 (0.03902883518610202) + | > loss_mel_ce: 2.9002525806427 (3.331019296287195) + | > loss: 0.733405590057373 (0.842512032837363) + | > current_lr: 5e-06 + | > step_time: 0.1571 (0.24505259403984667) + | > loader_time: 0.0102 (0.01168387176071259) + + + --> TIME: 2024-10-20 05:47:08 -- STEP: 65150/69059 -- GLOBAL_STEP: 65150 + | > loss_text_ce: 0.03060382790863514 (0.03902396569904739) + | > loss_mel_ce: 2.7792556285858154 (3.33083725865949) + | > loss: 0.7024648785591125 (0.8424653060630211) + | > current_lr: 5e-06 + | > step_time: 0.1789 (0.24504799738171826) + | > loader_time: 0.0107 (0.011683725110403118) + + + --> TIME: 2024-10-20 05:47:27 -- STEP: 65200/69059 -- GLOBAL_STEP: 65200 + | > loss_text_ce: 0.0329492911696434 (0.039019010333326755) + | > loss_mel_ce: 3.070209264755249 (3.330703966471302) + | > loss: 0.7757896184921265 (0.8424307441784566) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.2450450143748266) + | > loader_time: 0.0121 (0.011683700786777784) + + + --> TIME: 2024-10-20 05:47:45 -- STEP: 65250/69059 -- GLOBAL_STEP: 65250 + | > loss_text_ce: 0.033845748752355576 (0.039014048885082614) + | > loss_mel_ce: 3.238149642944336 (3.330546471212089) + | > loss: 0.8179988265037537 (0.8423901300019238) + | > current_lr: 5e-06 + | > step_time: 0.1955 (0.24503954859985702) + | > loader_time: 0.0124 (0.011683663338993684) + + + --> TIME: 2024-10-20 05:48:02 -- STEP: 65300/69059 -- GLOBAL_STEP: 65300 + | > loss_text_ce: 0.035509686917066574 (0.03900945542765249) + | > loss_mel_ce: 3.0183756351470947 (3.3303875788685606) + | > loss: 0.7634713053703308 (0.8423492585508242) + | > current_lr: 5e-06 + | > step_time: 0.2047 (0.24503071634911833) + | > loader_time: 0.0104 (0.011683392626950772) + + + --> TIME: 2024-10-20 05:48:21 -- STEP: 65350/69059 -- GLOBAL_STEP: 65350 + | > loss_text_ce: 0.02970711700618267 (0.03900445013813624) + | > loss_mel_ce: 3.1147305965423584 (3.330211245490092) + | > loss: 0.786109447479248 (0.8423039238819575) + | > current_lr: 5e-06 + | > step_time: 0.1714 (0.2450374196372875) + | > loader_time: 0.0104 (0.011683435961775117) + + + --> TIME: 2024-10-20 05:48:40 -- STEP: 65400/69059 -- GLOBAL_STEP: 65400 + | > loss_text_ce: 0.029965030029416084 (0.0389998507837188) + | > loss_mel_ce: 3.1857800483703613 (3.330071084313423) + | > loss: 0.803936243057251 (0.842267733746713) + | > current_lr: 5e-06 + | > step_time: 0.1512 (0.24504223488521867) + | > loader_time: 0.0099 (0.011683232117136667) + + + --> TIME: 2024-10-20 05:48:59 -- STEP: 65450/69059 -- GLOBAL_STEP: 65450 + | > loss_text_ce: 0.03071807138621807 (0.0389949575638589) + | > loss_mel_ce: 3.072282314300537 (3.329886908440107) + | > loss: 0.7757501006126404 (0.8422204664738883) + | > current_lr: 5e-06 + | > step_time: 0.1767 (0.24504431322023465) + | > loader_time: 0.0102 (0.011683358993122266) + + + --> TIME: 2024-10-20 05:49:17 -- STEP: 65500/69059 -- GLOBAL_STEP: 65500 + | > loss_text_ce: 0.036638956516981125 (0.03899027699758147) + | > loss_mel_ce: 3.478781223297119 (3.3297198604590887) + | > loss: 0.8788550496101379 (0.8421775343354428) + | > current_lr: 5e-06 + | > step_time: 0.3599 (0.24503141685296562) + | > loader_time: 0.0125 (0.0116830659167457) + + + --> TIME: 2024-10-20 05:49:34 -- STEP: 65550/69059 -- GLOBAL_STEP: 65550 + | > loss_text_ce: 0.03286101669073105 (0.03898561480755552) + | > loss_mel_ce: 3.044872522354126 (3.3295349062862014) + | > loss: 0.7694333791732788 (0.8421301302457026) + | > current_lr: 5e-06 + | > step_time: 0.197 (0.2450226362418983) + | > loader_time: 0.0113 (0.011682955271228392) + + + --> TIME: 2024-10-20 05:49:53 -- STEP: 65600/69059 -- GLOBAL_STEP: 65600 + | > loss_text_ce: 0.030320866033434868 (0.03898088310643555) + | > loss_mel_ce: 3.546586036682129 (3.329382235949346) + | > loss: 0.8942267298698425 (0.8420907797346397) + | > current_lr: 5e-06 + | > step_time: 0.4535 (0.24501832915151991) + | > loader_time: 0.0146 (0.01168264233847942) + + + --> TIME: 2024-10-20 05:50:13 -- STEP: 65650/69059 -- GLOBAL_STEP: 65650 + | > loss_text_ce: 0.03137097880244255 (0.03897627377759136) + | > loss_mel_ce: 2.8998208045959473 (3.3292450165948204) + | > loss: 0.7327979207038879 (0.8420553225628409) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.245040376892729) + | > loader_time: 0.0104 (0.011682774532140873) + + + --> TIME: 2024-10-20 05:50:31 -- STEP: 65700/69059 -- GLOBAL_STEP: 65700 + | > loss_text_ce: 0.03174019977450371 (0.03897125399531114) + | > loss_mel_ce: 3.290306329727173 (3.329118572936197) + | > loss: 0.8305116295814514 (0.8420224567054801) + | > current_lr: 5e-06 + | > step_time: 0.2759 (0.24503576896324913) + | > loader_time: 0.0117 (0.011682473408395642) + + + --> TIME: 2024-10-20 05:50:49 -- STEP: 65750/69059 -- GLOBAL_STEP: 65750 + | > loss_text_ce: 0.03399623930454254 (0.038966437119012785) + | > loss_mel_ce: 3.1305079460144043 (3.3289685900546617) + | > loss: 0.7911260724067688 (0.8419837567661205) + | > current_lr: 5e-06 + | > step_time: 0.1957 (0.2450274868446611) + | > loader_time: 0.0118 (0.011682115910171086) + + + --> TIME: 2024-10-20 05:51:07 -- STEP: 65800/69059 -- GLOBAL_STEP: 65800 + | > loss_text_ce: 0.030587607994675636 (0.03896161038363393) + | > loss_mel_ce: 3.0643506050109863 (3.3288160529361175) + | > loss: 0.7737345695495605 (0.8419444158030143) + | > current_lr: 5e-06 + | > step_time: 0.156 (0.24502776977501378) + | > loader_time: 0.0108 (0.011681858163474161) + + + --> TIME: 2024-10-20 05:51:25 -- STEP: 65850/69059 -- GLOBAL_STEP: 65850 + | > loss_text_ce: 0.030036887153983116 (0.03895668869177592) + | > loss_mel_ce: 2.9703633785247803 (3.3286235695058912) + | > loss: 0.7501000761985779 (0.841895064522846) + | > current_lr: 5e-06 + | > step_time: 0.1682 (0.24502027587992004) + | > loader_time: 0.0107 (0.011681543761523437) + + + --> TIME: 2024-10-20 05:51:44 -- STEP: 65900/69059 -- GLOBAL_STEP: 65900 + | > loss_text_ce: 0.035253796726465225 (0.03895199915939652) + | > loss_mel_ce: 3.2669923305511475 (3.328456033784798) + | > loss: 0.8255615234375 (0.841852008211957) + | > current_lr: 5e-06 + | > step_time: 0.234 (0.2450216978562258) + | > loader_time: 0.0109 (0.011681188610869926) + + + --> TIME: 2024-10-20 05:52:01 -- STEP: 65950/69059 -- GLOBAL_STEP: 65950 + | > loss_text_ce: 0.03362572193145752 (0.03894711585027646) + | > loss_mel_ce: 3.097036838531494 (3.3282790587832163) + | > loss: 0.7826656103134155 (0.8418065436352742) + | > current_lr: 5e-06 + | > step_time: 0.3127 (0.2450026102478166) + | > loader_time: 0.0105 (0.011680724326185044) + + + --> TIME: 2024-10-20 05:52:19 -- STEP: 66000/69059 -- GLOBAL_STEP: 66000 + | > loss_text_ce: 0.03363263979554176 (0.03894220476069755) + | > loss_mel_ce: 3.4140987396240234 (3.3280990661346417) + | > loss: 0.8619328737258911 (0.8417603176994762) + | > current_lr: 5e-06 + | > step_time: 0.3417 (0.24499890545642738) + | > loader_time: 0.012 (0.011680587739655447) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_66000.pth + + --> TIME: 2024-10-20 05:52:53 -- STEP: 66050/69059 -- GLOBAL_STEP: 66050 + | > loss_text_ce: 0.03337475657463074 (0.03893774372194819) + | > loss_mel_ce: 3.6054558753967285 (3.3279517176221534) + | > loss: 0.9097076654434204 (0.8417223653127357) + | > current_lr: 5e-06 + | > step_time: 0.4342 (0.24501309872034552) + | > loader_time: 0.0136 (0.01168069652135762) + + + --> TIME: 2024-10-20 05:53:12 -- STEP: 66100/69059 -- GLOBAL_STEP: 66100 + | > loss_text_ce: 0.03218480572104454 (0.03893292855302254) + | > loss_mel_ce: 3.3147082328796387 (3.3277720695869153) + | > loss: 0.836723268032074 (0.8416762495140091) + | > current_lr: 5e-06 + | > step_time: 0.3746 (0.24501183558159623) + | > loader_time: 0.0151 (0.01168091025186557) + + + --> TIME: 2024-10-20 05:53:30 -- STEP: 66150/69059 -- GLOBAL_STEP: 66150 + | > loss_text_ce: 0.03126092255115509 (0.03892797427802379) + | > loss_mel_ce: 3.0629284381866455 (3.3276410373880796) + | > loss: 0.773547351360321 (0.8416422528970096) + | > current_lr: 5e-06 + | > step_time: 0.2238 (0.245010951518652) + | > loader_time: 0.0107 (0.011680887143718212) + + + --> TIME: 2024-10-20 05:53:49 -- STEP: 66200/69059 -- GLOBAL_STEP: 66200 + | > loss_text_ce: 0.029973695054650307 (0.038923403350185744) + | > loss_mel_ce: 2.5803637504577637 (3.327523934142454) + | > loss: 0.6525843739509583 (0.8416118343521108) + | > current_lr: 5e-06 + | > step_time: 0.1653 (0.24500771503434074) + | > loader_time: 0.0111 (0.011680864030861039) + + + --> TIME: 2024-10-20 05:54:06 -- STEP: 66250/69059 -- GLOBAL_STEP: 66250 + | > loss_text_ce: 0.031800251454114914 (0.038918532474816546) + | > loss_mel_ce: 3.088549852371216 (3.3273556054996862) + | > loss: 0.7800875306129456 (0.841568534470505) + | > current_lr: 5e-06 + | > step_time: 0.2018 (0.24499041159287935) + | > loader_time: 0.011 (0.011680709572558108) + + + --> TIME: 2024-10-20 05:54:24 -- STEP: 66300/69059 -- GLOBAL_STEP: 66300 + | > loss_text_ce: 0.033214021474123 (0.03891371881735341) + | > loss_mel_ce: 3.443148374557495 (3.3271972160749175) + | > loss: 0.8690906167030334 (0.8415277336976115) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24499197119860985) + | > loader_time: 0.0111 (0.011680809976828016) + + + --> TIME: 2024-10-20 05:54:42 -- STEP: 66350/69059 -- GLOBAL_STEP: 66350 + | > loss_text_ce: 0.030878324061632156 (0.03890912308137942) + | > loss_mel_ce: 2.8351869583129883 (3.327021059364674) + | > loss: 0.7165163159370422 (0.8414825455865745) + | > current_lr: 5e-06 + | > step_time: 0.202 (0.2449824433409368) + | > loader_time: 0.011 (0.011680671272191566) + + + --> TIME: 2024-10-20 05:55:00 -- STEP: 66400/69059 -- GLOBAL_STEP: 66400 + | > loss_text_ce: 0.03260461613535881 (0.03890416854571688) + | > loss_mel_ce: 3.4888756275177 (3.3267926935510332) + | > loss: 0.8803700804710388 (0.841424215500614) + | > current_lr: 5e-06 + | > step_time: 0.3346 (0.2449783838297947) + | > loader_time: 0.0122 (0.011680684502584363) + + + --> TIME: 2024-10-20 05:55:19 -- STEP: 66450/69059 -- GLOBAL_STEP: 66450 + | > loss_text_ce: 0.032422591000795364 (0.03889973043422736) + | > loss_mel_ce: 2.994410514831543 (3.326669068027911) + | > loss: 0.7567082643508911 (0.841392199592182) + | > current_lr: 5e-06 + | > step_time: 0.1781 (0.2449895857688266) + | > loader_time: 0.0113 (0.011680788979483667) + + + --> TIME: 2024-10-20 05:55:38 -- STEP: 66500/69059 -- GLOBAL_STEP: 66500 + | > loss_text_ce: 0.036403339356184006 (0.038895118483895) + | > loss_mel_ce: 3.3209171295166016 (3.326512314441496) + | > loss: 0.8393301367759705 (0.8413518582098474) + | > current_lr: 5e-06 + | > step_time: 0.5155 (0.24498904126748103) + | > loader_time: 0.0151 (0.011680648362726174) + + + --> TIME: 2024-10-20 05:55:57 -- STEP: 66550/69059 -- GLOBAL_STEP: 66550 + | > loss_text_ce: 0.030327986925840378 (0.03889057380217907) + | > loss_mel_ce: 2.6890509128570557 (3.326362632454817) + | > loss: 0.6798447370529175 (0.841313301539798) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24499991015685751) + | > loader_time: 0.0104 (0.011680696467364464) + + + --> TIME: 2024-10-20 05:56:15 -- STEP: 66600/69059 -- GLOBAL_STEP: 66600 + | > loss_text_ce: 0.03556760400533676 (0.03888551472926711) + | > loss_mel_ce: 3.0378336906433105 (3.3262093626987346) + | > loss: 0.7683503031730652 (0.8412737193318823) + | > current_lr: 5e-06 + | > step_time: 0.1577 (0.24499022218916144) + | > loader_time: 0.0114 (0.011680720937144623) + + + --> TIME: 2024-10-20 05:56:33 -- STEP: 66650/69059 -- GLOBAL_STEP: 66650 + | > loss_text_ce: 0.03181828558444977 (0.03888106982554117) + | > loss_mel_ce: 3.3080248832702637 (3.326046630933768) + | > loss: 0.8349608182907104 (0.8412319251673621) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.2449828823473072) + | > loader_time: 0.0105 (0.011680633290227133) + + + --> TIME: 2024-10-20 05:56:52 -- STEP: 66700/69059 -- GLOBAL_STEP: 66700 + | > loss_text_ce: 0.03330875188112259 (0.03887659936461897) + | > loss_mel_ce: 3.2029380798339844 (3.325945927832246) + | > loss: 0.8090617060661316 (0.8412056317760143) + | > current_lr: 5e-06 + | > step_time: 0.3445 (0.24499384827878343) + | > loader_time: 0.0127 (0.011680523616441875) + + + --> TIME: 2024-10-20 05:57:10 -- STEP: 66750/69059 -- GLOBAL_STEP: 66750 + | > loss_text_ce: 0.03170323744416237 (0.03887170646529367) + | > loss_mel_ce: 3.3837130069732666 (3.325774015215886) + | > loss: 0.8538540601730347 (0.8411614303963918) + | > current_lr: 5e-06 + | > step_time: 0.2623 (0.24497948878713316) + | > loader_time: 0.0115 (0.011680276149221099) + + + --> TIME: 2024-10-20 05:57:30 -- STEP: 66800/69059 -- GLOBAL_STEP: 66800 + | > loss_text_ce: 0.03461029753088951 (0.038867087478018494) + | > loss_mel_ce: 3.1886794567108154 (3.3256528250292976) + | > loss: 0.8058224320411682 (0.8411299781037315) + | > current_lr: 5e-06 + | > step_time: 0.1919 (0.24500222361730245) + | > loader_time: 0.014 (0.011680703719932847) + + + --> TIME: 2024-10-20 05:57:49 -- STEP: 66850/69059 -- GLOBAL_STEP: 66850 + | > loss_text_ce: 0.03397921845316887 (0.0388622549454564) + | > loss_mel_ce: 3.2044970989227295 (3.3255000981806515) + | > loss: 0.8096190690994263 (0.8410905882593593) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24500291097850693) + | > loader_time: 0.0109 (0.011680376581407402) + + + --> TIME: 2024-10-20 05:58:06 -- STEP: 66900/69059 -- GLOBAL_STEP: 66900 + | > loss_text_ce: 0.03217478096485138 (0.038857321718036276) + | > loss_mel_ce: 3.2930116653442383 (3.325331900198173) + | > loss: 0.8312966227531433 (0.8410473054558363) + | > current_lr: 5e-06 + | > step_time: 0.1738 (0.24498785824697286) + | > loader_time: 0.011 (0.011680045737279345) + + + --> TIME: 2024-10-20 05:58:25 -- STEP: 66950/69059 -- GLOBAL_STEP: 66950 + | > loss_text_ce: 0.03145793452858925 (0.0388530242831746) + | > loss_mel_ce: 2.9888272285461426 (3.325167350683563) + | > loss: 0.7550712823867798 (0.8410050937183955) + | > current_lr: 5e-06 + | > step_time: 0.1759 (0.24499056213555534) + | > loader_time: 0.0104 (0.011680049112788568) + + + --> TIME: 2024-10-20 05:58:44 -- STEP: 67000/69059 -- GLOBAL_STEP: 67000 + | > loss_text_ce: 0.03226497769355774 (0.03884860497418361) + | > loss_mel_ce: 3.014589786529541 (3.325030462460719) + | > loss: 0.7617136836051941 (0.8409697668347799) + | > current_lr: 5e-06 + | > step_time: 0.1636 (0.24500022459741844) + | > loader_time: 0.0113 (0.011679967997679045) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_67000.pth + + --> TIME: 2024-10-20 05:59:17 -- STEP: 67050/69059 -- GLOBAL_STEP: 67050 + | > loss_text_ce: 0.03168940916657448 (0.03884383178888979) + | > loss_mel_ce: 3.417623996734619 (3.3248784376772953) + | > loss: 0.8623283505439758 (0.8409305673415419) + | > current_lr: 5e-06 + | > step_time: 0.2611 (0.24498974960122327) + | > loader_time: 0.0122 (0.011679768196066106) + + + --> TIME: 2024-10-20 05:59:35 -- STEP: 67100/69059 -- GLOBAL_STEP: 67100 + | > loss_text_ce: 0.033122532069683075 (0.03883919219132446) + | > loss_mel_ce: 3.0200140476226807 (3.3247331989042324) + | > loss: 0.7632841467857361 (0.8408930977508328) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.2449873364735644) + | > loader_time: 0.0109 (0.011679676719642764) + + + --> TIME: 2024-10-20 05:59:55 -- STEP: 67150/69059 -- GLOBAL_STEP: 67150 + | > loss_text_ce: 0.03139251098036766 (0.03883458319055578) + | > loss_mel_ce: 3.086425542831421 (3.324577901702437) + | > loss: 0.7794545292854309 (0.8408531211990092) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24500051497702974) + | > loader_time: 0.0106 (0.011679771853664941) + + + --> TIME: 2024-10-20 06:00:14 -- STEP: 67200/69059 -- GLOBAL_STEP: 67200 + | > loss_text_ce: 0.033706121146678925 (0.038830031637611985) + | > loss_mel_ce: 2.756434679031372 (3.324413554281694) + | > loss: 0.6975352168083191 (0.840810896453582) + | > current_lr: 5e-06 + | > step_time: 0.1555 (0.24500337012466925) + | > loader_time: 0.0106 (0.011679794500981038) + + + --> TIME: 2024-10-20 06:00:31 -- STEP: 67250/69059 -- GLOBAL_STEP: 67250 + | > loss_text_ce: 0.033002544194459915 (0.03882531720241943) + | > loss_mel_ce: 3.563290596008301 (3.324254839890054) + | > loss: 0.8990733027458191 (0.8407700392464295) + | > current_lr: 5e-06 + | > step_time: 0.2222 (0.24499183988039372) + | > loader_time: 0.0112 (0.011679987577700683) + + + --> TIME: 2024-10-20 06:00:52 -- STEP: 67300/69059 -- GLOBAL_STEP: 67300 + | > loss_text_ce: 0.03163152560591698 (0.038820809183771954) + | > loss_mel_ce: 3.353020191192627 (3.3241397840767593) + | > loss: 0.8461629152297974 (0.8407401482878271) + | > current_lr: 5e-06 + | > step_time: 0.3355 (0.24501422409494064) + | > loader_time: 0.0124 (0.011680165453084668) + + + --> TIME: 2024-10-20 06:01:10 -- STEP: 67350/69059 -- GLOBAL_STEP: 67350 + | > loss_text_ce: 0.032914262264966965 (0.038816316027467664) + | > loss_mel_ce: 2.896634578704834 (3.3239925368202003) + | > loss: 0.7323871850967407 (0.8407022131861113) + | > current_lr: 5e-06 + | > step_time: 0.1865 (0.24501041625107134) + | > loader_time: 0.011 (0.011680132470134436) + + + --> TIME: 2024-10-20 06:01:29 -- STEP: 67400/69059 -- GLOBAL_STEP: 67400 + | > loss_text_ce: 0.034916117787361145 (0.0388117580842057) + | > loss_mel_ce: 2.7862677574157715 (3.3238495336052813) + | > loss: 0.705295979976654 (0.8406653228976974) + | > current_lr: 5e-06 + | > step_time: 0.1636 (0.24501265100980013) + | > loader_time: 0.0109 (0.011680242732299869) + + + --> TIME: 2024-10-20 06:01:48 -- STEP: 67450/69059 -- GLOBAL_STEP: 67450 + | > loss_text_ce: 0.03372896462678909 (0.03880750546760658) + | > loss_mel_ce: 2.937267541885376 (3.323695955810225) + | > loss: 0.7427491545677185 (0.8406258652950779) + | > current_lr: 5e-06 + | > step_time: 0.1958 (0.2450135780741675) + | > loader_time: 0.0112 (0.011680227057681045) + + + --> TIME: 2024-10-20 06:02:05 -- STEP: 67500/69059 -- GLOBAL_STEP: 67500 + | > loss_text_ce: 0.02972014993429184 (0.03880291705597846) + | > loss_mel_ce: 3.2044739723205566 (3.3235565151779634) + | > loss: 0.8085485100746155 (0.8405898580356895) + | > current_lr: 5e-06 + | > step_time: 0.1557 (0.24500158571313924) + | > loader_time: 0.0103 (0.011679827230947964) + + + --> TIME: 2024-10-20 06:02:25 -- STEP: 67550/69059 -- GLOBAL_STEP: 67550 + | > loss_text_ce: 0.03055763617157936 (0.03879854111589571) + | > loss_mel_ce: 2.9160866737365723 (3.323437049958891) + | > loss: 0.7366610765457153 (0.8405588977456903) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24501173477539914) + | > loader_time: 0.0103 (0.011679841499695656) + + + --> TIME: 2024-10-20 06:02:44 -- STEP: 67600/69059 -- GLOBAL_STEP: 67600 + | > loss_text_ce: 0.034396808594465256 (0.038794149511587085) + | > loss_mel_ce: 2.8118648529052734 (3.323288553096123) + | > loss: 0.7115654349327087 (0.8405206756284974) + | > current_lr: 5e-06 + | > step_time: 0.19 (0.24502299267864786) + | > loader_time: 0.0108 (0.011679876347970651) + + + --> TIME: 2024-10-20 06:03:02 -- STEP: 67650/69059 -- GLOBAL_STEP: 67650 + | > loss_text_ce: 0.033588193356990814 (0.03878924581043923) + | > loss_mel_ce: 3.2832629680633545 (3.323122846105753) + | > loss: 0.8292127847671509 (0.8404780229558276) + | > current_lr: 5e-06 + | > step_time: 0.2585 (0.24501125530586887) + | > loader_time: 0.0118 (0.011679662942710113) + + + --> TIME: 2024-10-20 06:03:21 -- STEP: 67700/69059 -- GLOBAL_STEP: 67700 + | > loss_text_ce: 0.032088931649923325 (0.0387845907073074) + | > loss_mel_ce: 3.475959300994873 (3.3229939206769834) + | > loss: 0.8770120739936829 (0.8404446278259369) + | > current_lr: 5e-06 + | > step_time: 0.2984 (0.2450148434829148) + | > loader_time: 0.0124 (0.01167950438960086) + + + --> TIME: 2024-10-20 06:03:39 -- STEP: 67750/69059 -- GLOBAL_STEP: 67750 + | > loss_text_ce: 0.03396515175700188 (0.03878019043415532) + | > loss_mel_ce: 3.370549201965332 (3.322833005373759) + | > loss: 0.8511285781860352 (0.8404032989304898) + | > current_lr: 5e-06 + | > step_time: 0.2029 (0.2450210351908778) + | > loader_time: 0.0123 (0.011679752135188797) + + + --> TIME: 2024-10-20 06:03:58 -- STEP: 67800/69059 -- GLOBAL_STEP: 67800 + | > loss_text_ce: 0.028690963983535767 (0.03877576530520885) + | > loss_mel_ce: 3.227776288986206 (3.322696833364371) + | > loss: 0.8141168355941772 (0.840368149644507) + | > current_lr: 5e-06 + | > step_time: 0.1709 (0.24501809618114365) + | > loader_time: 0.0105 (0.011679523887887443) + + + --> TIME: 2024-10-20 06:04:18 -- STEP: 67850/69059 -- GLOBAL_STEP: 67850 + | > loss_text_ce: 0.032180558890104294 (0.03877143246598659) + | > loss_mel_ce: 3.128340005874634 (3.3225520422780077) + | > loss: 0.7901301383972168 (0.8403308686660172) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.24503927850688123) + | > loader_time: 0.0104 (0.011679700782567836) + + + --> TIME: 2024-10-20 06:04:37 -- STEP: 67900/69059 -- GLOBAL_STEP: 67900 + | > loss_text_ce: 0.03314831107854843 (0.03876699009342349) + | > loss_mel_ce: 2.995182752609253 (3.3224215896982763) + | > loss: 0.7570827603340149 (0.840297144927931) + | > current_lr: 5e-06 + | > step_time: 0.2873 (0.2450383614826623) + | > loader_time: 0.0114 (0.011679549603469225) + + + --> TIME: 2024-10-20 06:04:54 -- STEP: 67950/69059 -- GLOBAL_STEP: 67950 + | > loss_text_ce: 0.0339939221739769 (0.03876277575733945) + | > loss_mel_ce: 3.045450210571289 (3.322255873578352) + | > loss: 0.7698610424995422 (0.8402546623149647) + | > current_lr: 5e-06 + | > step_time: 0.1872 (0.24502941721761107) + | > loader_time: 0.0116 (0.011679399067904673) + + + --> TIME: 2024-10-20 06:05:13 -- STEP: 68000/69059 -- GLOBAL_STEP: 68000 + | > loss_text_ce: 0.03163871541619301 (0.038757990666171784) + | > loss_mel_ce: 2.9744298458099365 (3.322115841476347) + | > loss: 0.751517117023468 (0.8402184580178836) + | > current_lr: 5e-06 + | > step_time: 0.1576 (0.24502728776370772) + | > loader_time: 0.0105 (0.011679210862692647) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_68000.pth + + --> TIME: 2024-10-20 06:05:47 -- STEP: 68050/69059 -- GLOBAL_STEP: 68050 + | > loss_text_ce: 0.03291837126016617 (0.038753517541451066) + | > loss_mel_ce: 3.0373568534851074 (3.321996819747002) + | > loss: 0.767568826675415 (0.8401875843062824) + | > current_lr: 5e-06 + | > step_time: 0.1948 (0.24504286490670898) + | > loader_time: 0.0119 (0.011679175900326832) + + + --> TIME: 2024-10-20 06:06:06 -- STEP: 68100/69059 -- GLOBAL_STEP: 68100 + | > loss_text_ce: 0.030838651582598686 (0.03874913698040389) + | > loss_mel_ce: 2.986598491668701 (3.321815510842934) + | > loss: 0.7543593049049377 (0.8401411619392974) + | > current_lr: 5e-06 + | > step_time: 0.163 (0.24504200970723936) + | > loader_time: 0.0102 (0.011678981182333638) + + + --> TIME: 2024-10-20 06:06:25 -- STEP: 68150/69059 -- GLOBAL_STEP: 68150 + | > loss_text_ce: 0.033897437155246735 (0.03874456651713763) + | > loss_mel_ce: 3.0358316898345947 (3.3216369876105825) + | > loss: 0.7674322724342346 (0.8400953885176049) + | > current_lr: 5e-06 + | > step_time: 0.2786 (0.2450465747283297) + | > loader_time: 0.0112 (0.011679040901116893) + + + --> TIME: 2024-10-20 06:06:44 -- STEP: 68200/69059 -- GLOBAL_STEP: 68200 + | > loss_text_ce: 0.031311314553022385 (0.038740025130195824) + | > loss_mel_ce: 3.585163116455078 (3.321498557482987) + | > loss: 0.9041185975074768 (0.840059645642634) + | > current_lr: 5e-06 + | > step_time: 0.3113 (0.2450461561588947) + | > loader_time: 0.0133 (0.0116807545105383) + + + --> TIME: 2024-10-20 06:07:01 -- STEP: 68250/69059 -- GLOBAL_STEP: 68250 + | > loss_text_ce: 0.03578934445977211 (0.03873559372279858) + | > loss_mel_ce: 2.693633556365967 (3.321336913472121) + | > loss: 0.6823557019233704 (0.8400181267864113) + | > current_lr: 5e-06 + | > step_time: 0.1851 (0.24503222458877838) + | > loader_time: 0.0109 (0.011680365192147824) + + + --> TIME: 2024-10-20 06:07:19 -- STEP: 68300/69059 -- GLOBAL_STEP: 68300 + | > loss_text_ce: 0.03313908725976944 (0.03873099847387159) + | > loss_mel_ce: 2.913764238357544 (3.321148046026458) + | > loss: 0.7367258071899414 (0.8399697611124254) + | > current_lr: 5e-06 + | > step_time: 0.1731 (0.24502505904584462) + | > loader_time: 0.0113 (0.01168006181367842) + + + --> TIME: 2024-10-20 06:07:37 -- STEP: 68350/69059 -- GLOBAL_STEP: 68350 + | > loss_text_ce: 0.03416043519973755 (0.0387265044317224) + | > loss_mel_ce: 3.0868515968322754 (3.3210097720907212) + | > loss: 0.780252993106842 (0.8399340691170569) + | > current_lr: 5e-06 + | > step_time: 0.1939 (0.24502398691477348) + | > loader_time: 0.0108 (0.011680126228555748) + + + --> TIME: 2024-10-20 06:07:56 -- STEP: 68400/69059 -- GLOBAL_STEP: 68400 + | > loss_text_ce: 0.03139040991663933 (0.038721960288423174) + | > loss_mel_ce: 2.978390693664551 (3.3208675973422253) + | > loss: 0.7524452805519104 (0.8398973893927562) + | > current_lr: 5e-06 + | > step_time: 0.243 (0.24502537462097854) + | > loader_time: 0.0116 (0.011680078530869272) + + + --> TIME: 2024-10-20 06:08:15 -- STEP: 68450/69059 -- GLOBAL_STEP: 68450 + | > loss_text_ce: 0.033530060201883316 (0.03871748260723724) + | > loss_mel_ce: 3.5540823936462402 (3.3207433210105335) + | > loss: 0.8969030976295471 (0.8398652008913843) + | > current_lr: 5e-06 + | > step_time: 0.3932 (0.24503104555075664) + | > loader_time: 0.0143 (0.01167993818607705) + + + --> TIME: 2024-10-20 06:08:34 -- STEP: 68500/69059 -- GLOBAL_STEP: 68500 + | > loss_text_ce: 0.03264712914824486 (0.03871325972730634) + | > loss_mel_ce: 2.885983467102051 (3.3205836923905445) + | > loss: 0.7296576499938965 (0.8398242380157883) + | > current_lr: 5e-06 + | > step_time: 0.1727 (0.2450353073029622) + | > loader_time: 0.0106 (0.011679938974171642) + + + --> TIME: 2024-10-20 06:08:52 -- STEP: 68550/69059 -- GLOBAL_STEP: 68550 + | > loss_text_ce: 0.03471153974533081 (0.038709035791989005) + | > loss_mel_ce: 3.1365983486175537 (3.320430096866952) + | > loss: 0.7928274869918823 (0.8397847831492711) + | > current_lr: 5e-06 + | > step_time: 0.3798 (0.2450323551714985) + | > loader_time: 0.0135 (0.011679743736178036) + + + --> TIME: 2024-10-20 06:09:10 -- STEP: 68600/69059 -- GLOBAL_STEP: 68600 + | > loss_text_ce: 0.03234098479151726 (0.03870469914103015) + | > loss_mel_ce: 3.2021431922912598 (3.3202923452541278) + | > loss: 0.8086210489273071 (0.8397492610852169) + | > current_lr: 5e-06 + | > step_time: 0.1728 (0.2450290224538252) + | > loader_time: 0.0113 (0.011679702875565494) + + + --> TIME: 2024-10-20 06:09:29 -- STEP: 68650/69059 -- GLOBAL_STEP: 68650 + | > loss_text_ce: 0.03220769390463829 (0.0387004687638716) + | > loss_mel_ce: 3.2709083557128906 (3.320184056402895) + | > loss: 0.8257790207862854 (0.8397211312813576) + | > current_lr: 5e-06 + | > step_time: 0.1943 (0.2450295671301867) + | > loader_time: 0.0108 (0.011679331639867048) + + + --> TIME: 2024-10-20 06:09:47 -- STEP: 68700/69059 -- GLOBAL_STEP: 68700 + | > loss_text_ce: 0.03191448375582695 (0.03869598689894374) + | > loss_mel_ce: 3.0579817295074463 (3.320015779416107) + | > loss: 0.7724740505218506 (0.8396779415666464) + | > current_lr: 5e-06 + | > step_time: 0.1969 (0.24502625880109513) + | > loader_time: 0.0108 (0.011679246283340994) + + + --> TIME: 2024-10-20 06:10:07 -- STEP: 68750/69059 -- GLOBAL_STEP: 68750 + | > loss_text_ce: 0.03256278485059738 (0.03869195772525911) + | > loss_mel_ce: 3.262660264968872 (3.319883959985207) + | > loss: 0.8238057494163513 (0.8396439794132939) + | > current_lr: 5e-06 + | > step_time: 0.5246 (0.2450475693928111) + | > loader_time: 0.0143 (0.011679273400740176) + + + --> TIME: 2024-10-20 06:10:28 -- STEP: 68800/69059 -- GLOBAL_STEP: 68800 + | > loss_text_ce: 0.02985832653939724 (0.03868777353267785) + | > loss_mel_ce: 3.178020477294922 (3.319748146807725) + | > loss: 0.8019697070121765 (0.8396089800689815) + | > current_lr: 5e-06 + | > step_time: 0.1486 (0.24506964307191756) + | > loader_time: 0.0108 (0.01167920409940008) + + + --> TIME: 2024-10-20 06:10:47 -- STEP: 68850/69059 -- GLOBAL_STEP: 68850 + | > loss_text_ce: 0.03242957592010498 (0.03868340083821962) + | > loss_mel_ce: 3.157620906829834 (3.3196171590497476) + | > loss: 0.7975126504898071 (0.8395751399567976) + | > current_lr: 5e-06 + | > step_time: 0.1762 (0.24507681463971545) + | > loader_time: 0.0105 (0.011679191108016563) + + + --> TIME: 2024-10-20 06:11:05 -- STEP: 68900/69059 -- GLOBAL_STEP: 68900 + | > loss_text_ce: 0.0349445566534996 (0.03867890025887757) + | > loss_mel_ce: 3.5313103199005127 (3.3194322887733403) + | > loss: 0.8915637135505676 (0.8395277972422086) + | > current_lr: 5e-06 + | > step_time: 0.4344 (0.24506980249255414) + | > loader_time: 0.0137 (0.011678960735116178) + + + --> TIME: 2024-10-20 06:11:24 -- STEP: 68950/69059 -- GLOBAL_STEP: 68950 + | > loss_text_ce: 0.03041405789554119 (0.038674779132635013) + | > loss_mel_ce: 2.976463556289673 (3.3192609615374273) + | > loss: 0.7517194151878357 (0.8394839351526465) + | > current_lr: 5e-06 + | > step_time: 0.2029 (0.2450714032973302) + | > loader_time: 0.0105 (0.011678676508404864) + + + --> TIME: 2024-10-20 06:11:42 -- STEP: 69000/69059 -- GLOBAL_STEP: 69000 + | > loss_text_ce: 0.030933033674955368 (0.03867028191404904) + | > loss_mel_ce: 3.0621583461761475 (3.3191066388634622) + | > loss: 0.7732728719711304 (0.8394442301798566) + | > current_lr: 5e-06 + | > step_time: 0.1571 (0.24507196628874622) + | > loader_time: 0.0101 (0.011678451196007085) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_69000.pth + + --> TIME: 2024-10-20 06:12:16 -- STEP: 69050/69059 -- GLOBAL_STEP: 69050 + | > loss_text_ce: 0.032531410455703735 (0.038665621787670856) + | > loss_mel_ce: 3.1649043560028076 (3.3189310440682176) + | > loss: 0.7993589639663696 (0.83939916644642) + | > current_lr: 5e-06 + | > step_time: 0.2339 (0.2450567463184939) + | > loader_time: 0.0109 (0.011678189768642705) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.33018381121880874 (+0) + | > avg_loss_text_ce: 0.03161565944415463 (+0) + | > avg_loss_mel_ce: 2.9523396825945776 (+0) + | > avg_loss: 2.983955342196043 (+0) + + > BEST MODEL : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/best_model_69059.pth + + > EPOCH: 1/5 + --> checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b + + > TRAINING (2024-10-20 07:17:01)  + + --> TIME: 2024-10-20 07:17:18 -- STEP: 41/69059 -- GLOBAL_STEP: 69100 + | > loss_text_ce: 0.032961197197437286 (0.03281089581730888) + | > loss_mel_ce: 3.2209606170654297 (3.1948759032458796) + | > loss: 0.8134804368019104 (0.8069216972444115) + | > current_lr: 5e-06 + | > step_time: 0.199 (0.24813029824233637) + | > loader_time: 0.0127 (0.015467824005499122) + + + --> TIME: 2024-10-20 07:17:38 -- STEP: 91/69059 -- GLOBAL_STEP: 69150 + | > loss_text_ce: 0.028908006846904755 (0.032789112017541144) + | > loss_mel_ce: 2.6775364875793457 (3.1505631850315976) + | > loss: 0.6766111254692078 (0.7958380737147488) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2423 (0.2582925413991068) + | > loader_time: 0.0153 (0.013763464414156403) + + + --> TIME: 2024-10-20 07:17:57 -- STEP: 141/69059 -- GLOBAL_STEP: 69200 + | > loss_text_ce: 0.03145940974354744 (0.03271438779824594) + | > loss_mel_ce: 3.0654497146606445 (3.1449607923521223) + | > loss: 0.7742272615432739 (0.7944187953117045) + | > current_lr: 5e-06 + | > step_time: 0.1718 (0.25650297158153323) + | > loader_time: 0.0103 (0.012965812750741944) + + + --> TIME: 2024-10-20 07:18:15 -- STEP: 191/69059 -- GLOBAL_STEP: 69250 + | > loss_text_ce: 0.03322022780776024 (0.032766105090292326) + | > loss_mel_ce: 3.220872640609741 (3.14493794091709) + | > loss: 0.8135232329368591 (0.7944260114774654) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2377 (0.2509424424296275) + | > loader_time: 0.0112 (0.01270269224156884) + + + --> TIME: 2024-10-20 07:18:33 -- STEP: 241/69059 -- GLOBAL_STEP: 69300 + | > loss_text_ce: 0.032548289746046066 (0.03276036926294016) + | > loss_mel_ce: 3.2552971839904785 (3.1448553271313426) + | > loss: 0.8219613432884216 (0.7944039232503329) + | > current_lr: 5e-06 + | > step_time: 0.2064 (0.24928406165348543) + | > loader_time: 0.0109 (0.012483896556236931) + + + --> TIME: 2024-10-20 07:18:51 -- STEP: 291/69059 -- GLOBAL_STEP: 69350 + | > loss_text_ce: 0.035178642719984055 (0.03278806641109006) + | > loss_mel_ce: 3.018242597579956 (3.1353278930244572) + | > loss: 0.7633553147315979 (0.7920289889643689) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4453 (0.24731795894321298) + | > loader_time: 0.0127 (0.012350354407661144) + + + --> TIME: 2024-10-20 07:19:08 -- STEP: 341/69059 -- GLOBAL_STEP: 69400 + | > loss_text_ce: 0.035492904484272 (0.03274294939383734) + | > loss_mel_ce: 3.142141580581665 (3.1266659005296553) + | > loss: 0.7944086194038391 (0.7898522113774875) + | > current_lr: 5e-06 + | > step_time: 0.3126 (0.2439219448223841) + | > loader_time: 0.0129 (0.012228737828319014) + + + --> TIME: 2024-10-20 07:19:27 -- STEP: 391/69059 -- GLOBAL_STEP: 69450 + | > loss_text_ce: 0.03184321150183678 (0.03271036418845587) + | > loss_mel_ce: 2.747739315032959 (3.1229983760267883) + | > loss: 0.6948956251144409 (0.7889271844988281) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2263 (0.24389967162285925) + | > loader_time: 0.0107 (0.012170878212775103) + + + --> TIME: 2024-10-20 07:19:44 -- STEP: 441/69059 -- GLOBAL_STEP: 69500 + | > loss_text_ce: 0.03137899190187454 (0.032723943574054205) + | > loss_mel_ce: 2.9583611488342285 (3.1222077827064365) + | > loss: 0.7474350333213806 (0.788732930916507) + | > current_lr: 5e-06 + | > step_time: 0.195 (0.24211727410487308) + | > loader_time: 0.0115 (0.012098141538312908) + + + --> TIME: 2024-10-20 07:20:02 -- STEP: 491/69059 -- GLOBAL_STEP: 69550 + | > loss_text_ce: 0.03274092078208923 (0.03271721812560704) + | > loss_mel_ce: 2.8111648559570312 (3.1136269409156383) + | > loss: 0.7109764218330383 (0.7865860392993916) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2095 (0.24155682604570253) + | > loader_time: 0.0107 (0.012054351585217253) + + + --> TIME: 2024-10-20 07:20:21 -- STEP: 541/69059 -- GLOBAL_STEP: 69600 + | > loss_text_ce: 0.03369423374533653 (0.032716582882150964) + | > loss_mel_ce: 3.2876381874084473 (3.1162490404026784) + | > loss: 0.8303331136703491 (0.7872414055681491) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.2415282779171815) + | > loader_time: 0.0124 (0.012048280613700028) + + + --> TIME: 2024-10-20 07:20:38 -- STEP: 591/69059 -- GLOBAL_STEP: 69650 + | > loss_text_ce: 0.03254824876785278 (0.032686096157519384) + | > loss_mel_ce: 2.70489239692688 (3.1159848803796146) + | > loss: 0.684360146522522 (0.7871677439830023) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2224 (0.24032049130667285) + | > loader_time: 0.0109 (0.012010581602299873) + + + --> TIME: 2024-10-20 07:20:56 -- STEP: 641/69059 -- GLOBAL_STEP: 69700 + | > loss_text_ce: 0.033500637859106064 (0.032699402628027736) + | > loss_mel_ce: 3.118210554122925 (3.1143179642800805) + | > loss: 0.7879278063774109 (0.7867543414676804) + | > current_lr: 5e-06 + | > step_time: 0.2656 (0.23962630608152488) + | > loader_time: 0.0116 (0.01195513365235231) + + + --> TIME: 2024-10-20 07:21:14 -- STEP: 691/69059 -- GLOBAL_STEP: 69750 + | > loss_text_ce: 0.03106846660375595 (0.03270340396573853) + | > loss_mel_ce: 3.0550942420959473 (3.110501990131981) + | > loss: 0.7715407013893127 (0.7858013479650973) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.256 (0.2395166115546882) + | > loader_time: 0.012 (0.011950760605366281) + + + --> TIME: 2024-10-20 07:21:34 -- STEP: 741/69059 -- GLOBAL_STEP: 69800 + | > loss_text_ce: 0.032368242740631104 (0.032676408416748674) + | > loss_mel_ce: 3.3570804595947266 (3.111412415137659) + | > loss: 0.8473621606826782 (0.7860222056648835) + | > current_lr: 5e-06 + | > step_time: 0.2257 (0.24168712117893976) + | > loader_time: 0.011 (0.012000372213551543) + + + --> TIME: 2024-10-20 07:21:51 -- STEP: 791/69059 -- GLOBAL_STEP: 69850 + | > loss_text_ce: 0.03384225815534592 (0.03264803623288452) + | > loss_mel_ce: 3.1940150260925293 (3.1092823703129295) + | > loss: 0.8069643378257751 (0.7854826014804481) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2269 (0.24043164874147976) + | > loader_time: 0.0122 (0.0119704609423612) + + + --> TIME: 2024-10-20 07:22:08 -- STEP: 841/69059 -- GLOBAL_STEP: 69900 + | > loss_text_ce: 0.03002561815083027 (0.03262053496745831) + | > loss_mel_ce: 2.974412202835083 (3.1065467852616293) + | > loss: 0.7511094808578491 (0.7847918298241644) + | > current_lr: 5e-06 + | > step_time: 0.1783 (0.23970137254803417) + | > loader_time: 0.0112 (0.01194700021097407) + + + --> TIME: 2024-10-20 07:22:26 -- STEP: 891/69059 -- GLOBAL_STEP: 69950 + | > loss_text_ce: 0.033230260014534 (0.03262648869443819) + | > loss_mel_ce: 3.0912301540374756 (3.104116555683809) + | > loss: 0.7811151146888733 (0.7841857608186145) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3246 (0.23907536475612928) + | > loader_time: 0.0106 (0.011928672608569274) + + + --> TIME: 2024-10-20 07:22:43 -- STEP: 941/69059 -- GLOBAL_STEP: 70000 + | > loss_text_ce: 0.028515901416540146 (0.03260688103268298) + | > loss_mel_ce: 3.0738375186920166 (3.1043530227527363) + | > loss: 0.77558833360672 (0.7842399756662652) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.23858404666280392) + | > loader_time: 0.0112 (0.011900098873628941) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_70000.pth + + --> TIME: 2024-10-20 07:23:17 -- STEP: 991/69059 -- GLOBAL_STEP: 70050 + | > loss_text_ce: 0.03678895905613899 (0.03263980068461864) + | > loss_mel_ce: 3.137070894241333 (3.104585569393263) + | > loss: 0.7934649586677551 (0.7843063422577414) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.216 (0.23975229431714348) + | > loader_time: 0.0102 (0.011922888991570492) + + + --> TIME: 2024-10-20 07:23:36 -- STEP: 1041/69059 -- GLOBAL_STEP: 70100 + | > loss_text_ce: 0.03166478872299194 (0.0326344052868234) + | > loss_mel_ce: 3.114370107650757 (3.105225141178979) + | > loss: 0.7865087389945984 (0.7844648864495997) + | > current_lr: 5e-06 + | > step_time: 0.2979 (0.2400223524036829) + | > loader_time: 0.0108 (0.01191093140674484) + + + --> TIME: 2024-10-20 07:23:53 -- STEP: 1091/69059 -- GLOBAL_STEP: 70150 + | > loss_text_ce: 0.030960559844970703 (0.032624917340445245) + | > loss_mel_ce: 3.0066518783569336 (3.1052777336673074) + | > loss: 0.7594031095504761 (0.7844756628044367) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1902 (0.23898505946018628) + | > loader_time: 0.0096 (0.011886849302856549) + + + --> TIME: 2024-10-20 07:24:10 -- STEP: 1141/69059 -- GLOBAL_STEP: 70200 + | > loss_text_ce: 0.033919814974069595 (0.032606373767153814) + | > loss_mel_ce: 3.143690586090088 (3.103300010604259) + | > loss: 0.7944025993347168 (0.7839765962960948) + | > current_lr: 5e-06 + | > step_time: 0.3142 (0.2383787602734712) + | > loader_time: 0.0131 (0.011859310185251049) + + + --> TIME: 2024-10-20 07:24:28 -- STEP: 1191/69059 -- GLOBAL_STEP: 70250 + | > loss_text_ce: 0.03411530330777168 (0.03259344594467095) + | > loss_mel_ce: 3.096858263015747 (3.1030287660539533) + | > loss: 0.7827433943748474 (0.783905553127116) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.23855576146860066) + | > loader_time: 0.01 (0.011855317002880169) + + + --> TIME: 2024-10-20 07:24:47 -- STEP: 1241/69059 -- GLOBAL_STEP: 70300 + | > loss_text_ce: 0.03438547998666763 (0.032592931609243536) + | > loss_mel_ce: 3.1279749870300293 (3.1036335011819607) + | > loss: 0.7905901074409485 (0.7840566081351376) + | > current_lr: 5e-06 + | > step_time: 0.41 (0.2392182427005937) + | > loader_time: 0.0139 (0.011869586349782378) + + + --> TIME: 2024-10-20 07:25:08 -- STEP: 1291/69059 -- GLOBAL_STEP: 70350 + | > loss_text_ce: 0.03119196556508541 (0.03258643443187613) + | > loss_mel_ce: 3.037989377975464 (3.106663573905538) + | > loss: 0.7672953605651855 (0.7848125021186201) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1996 (0.24036078744854325) + | > loader_time: 0.0129 (0.011892281051387513) + + + --> TIME: 2024-10-20 07:25:26 -- STEP: 1341/69059 -- GLOBAL_STEP: 70400 + | > loss_text_ce: 0.03381078317761421 (0.03259652492289526) + | > loss_mel_ce: 2.8620853424072266 (3.105014523670444) + | > loss: 0.7239740490913391 (0.7844027622167431) + | > current_lr: 5e-06 + | > step_time: 0.187 (0.2406475810663279) + | > loader_time: 0.0106 (0.011899268244914723) + + + --> TIME: 2024-10-20 07:25:45 -- STEP: 1391/69059 -- GLOBAL_STEP: 70450 + | > loss_text_ce: 0.031014276668429375 (0.03260778543210599) + | > loss_mel_ce: 3.2689406871795654 (3.1052802150665983) + | > loss: 0.8249887228012085 (0.7844720001189852) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1906 (0.24078823112910125) + | > loader_time: 0.0116 (0.011912884599094663) + + + --> TIME: 2024-10-20 07:26:03 -- STEP: 1441/69059 -- GLOBAL_STEP: 70500 + | > loss_text_ce: 0.03304557874798775 (0.03262080441149546) + | > loss_mel_ce: 3.349493980407715 (3.105223978500578) + | > loss: 0.8456348776817322 (0.7844611956873014) + | > current_lr: 5e-06 + | > step_time: 0.3062 (0.2410248702470831) + | > loader_time: 0.0143 (0.011911930763249909) + + + --> TIME: 2024-10-20 07:26:23 -- STEP: 1491/69059 -- GLOBAL_STEP: 70550 + | > loss_text_ce: 0.03125348314642906 (0.0326120326011913) + | > loss_mel_ce: 3.2768161296844482 (3.106066386384663) + | > loss: 0.8270174264907837 (0.7846696046596403) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4666 (0.24155741326365993) + | > loader_time: 0.0161 (0.011918849388598421) + + + --> TIME: 2024-10-20 07:26:40 -- STEP: 1541/69059 -- GLOBAL_STEP: 70600 + | > loss_text_ce: 0.030315935611724854 (0.03259998997147596) + | > loss_mel_ce: 2.914172410964966 (3.1058619039388233) + | > loss: 0.7361220717430115 (0.7846154733805748) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.24112153238943843) + | > loader_time: 0.0106 (0.01191636016815996) + + + --> TIME: 2024-10-20 07:27:00 -- STEP: 1591/69059 -- GLOBAL_STEP: 70650 + | > loss_text_ce: 0.029738161712884903 (0.03259899590040325) + | > loss_mel_ce: 2.808851957321167 (3.1055624988527137) + | > loss: 0.709647536277771 (0.7845403736751085) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3376 (0.2415705376043026) + | > loader_time: 0.0121 (0.011921601801979535) + + + --> TIME: 2024-10-20 07:27:19 -- STEP: 1641/69059 -- GLOBAL_STEP: 70700 + | > loss_text_ce: 0.03179427608847618 (0.03259940289691917) + | > loss_mel_ce: 3.2032299041748047 (3.105697302902566) + | > loss: 0.8087560534477234 (0.7845741764646272) + | > current_lr: 5e-06 + | > step_time: 0.2021 (0.24208735838523368) + | > loader_time: 0.0104 (0.0119264364097265) + + + --> TIME: 2024-10-20 07:27:38 -- STEP: 1691/69059 -- GLOBAL_STEP: 70750 + | > loss_text_ce: 0.0325159914791584 (0.032606242236690626) + | > loss_mel_ce: 2.9896631240844727 (3.104612394594568) + | > loss: 0.7555447816848755 (0.7843046592298449) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5393 (0.24243587886420034) + | > loader_time: 0.0161 (0.011928419471844481) + + + --> TIME: 2024-10-20 07:27:58 -- STEP: 1741/69059 -- GLOBAL_STEP: 70800 + | > loss_text_ce: 0.029913699254393578 (0.032586639248463736) + | > loss_mel_ce: 2.983363151550293 (3.1058376239401935) + | > loss: 0.7533192038536072 (0.7846060658856963) + | > current_lr: 5e-06 + | > step_time: 0.1877 (0.2429534727509579) + | > loader_time: 0.0104 (0.011935035364851044) + + + --> TIME: 2024-10-20 07:28:16 -- STEP: 1791/69059 -- GLOBAL_STEP: 70850 + | > loss_text_ce: 0.03160034120082855 (0.03258025211800511) + | > loss_mel_ce: 3.0101771354675293 (3.1057543274013106) + | > loss: 0.7604443430900574 (0.7845836449508093) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2162 (0.24296655899848704) + | > loader_time: 0.0115 (0.011933687610509064) + + + --> TIME: 2024-10-20 07:28:36 -- STEP: 1841/69059 -- GLOBAL_STEP: 70900 + | > loss_text_ce: 0.03271188214421272 (0.03258068887299326) + | > loss_mel_ce: 2.736098289489746 (3.106556864551459) + | > loss: 0.6922025680541992 (0.7847843884061952) + | > current_lr: 5e-06 + | > step_time: 0.1958 (0.24336992431632337) + | > loader_time: 0.0107 (0.011943058757792332) + + + --> TIME: 2024-10-20 07:28:56 -- STEP: 1891/69059 -- GLOBAL_STEP: 70950 + | > loss_text_ce: 0.03410763293504715 (0.03258831774700666) + | > loss_mel_ce: 3.037348508834839 (3.1094956932592366) + | > loss: 0.7678640484809875 (0.7855210027828349) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2045 (0.24398194501666023) + | > loader_time: 0.0117 (0.011947068255266636) + + + --> TIME: 2024-10-20 07:29:14 -- STEP: 1941/69059 -- GLOBAL_STEP: 71000 + | > loss_text_ce: 0.03258805349469185 (0.0325904089548672) + | > loss_mel_ce: 3.3301596641540527 (3.110047195573126) + | > loss: 0.8406869173049927 (0.7856594011809396) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.24372141205238843) + | > loader_time: 0.0105 (0.011937325053581058) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_71000.pth + + --> TIME: 2024-10-20 07:29:48 -- STEP: 1991/69059 -- GLOBAL_STEP: 71050 + | > loss_text_ce: 0.03464214503765106 (0.03259290310697804) + | > loss_mel_ce: 3.097979784011841 (3.1091659993517045) + | > loss: 0.7831555008888245 (0.7854397257687997) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4751 (0.24371164243823618) + | > loader_time: 0.013 (0.011933314985872082) + + + --> TIME: 2024-10-20 07:30:06 -- STEP: 2041/69059 -- GLOBAL_STEP: 71100 + | > loss_text_ce: 0.028771808370947838 (0.03259520320197134) + | > loss_mel_ce: 3.24761700630188 (3.1085438598668325) + | > loss: 0.8190972208976746 (0.7852847659663791) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.24355102531586367) + | > loader_time: 0.0108 (0.011933360480608035) + + + --> TIME: 2024-10-20 07:30:26 -- STEP: 2091/69059 -- GLOBAL_STEP: 71150 + | > loss_text_ce: 0.03273483365774155 (0.0325905017248327) + | > loss_mel_ce: 3.0739505290985107 (3.1085914207496805) + | > loss: 0.7766713500022888 (0.7852954808346454) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2081 (0.24403011097627528) + | > loader_time: 0.0112 (0.011938077875554181) + + + --> TIME: 2024-10-20 07:30:43 -- STEP: 2141/69059 -- GLOBAL_STEP: 71200 + | > loss_text_ce: 0.02934112213551998 (0.032590532457018324) + | > loss_mel_ce: 3.223719596862793 (3.1091197645035034) + | > loss: 0.8132652044296265 (0.7854275743519243) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24364278696230338) + | > loader_time: 0.0101 (0.011929716812233558) + + + --> TIME: 2024-10-20 07:31:03 -- STEP: 2191/69059 -- GLOBAL_STEP: 71250 + | > loss_text_ce: 0.030724501237273216 (0.03259802140067662) + | > loss_mel_ce: 2.7647616863250732 (3.109100400993454) + | > loss: 0.6988715529441833 (0.7854246057532576) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.284 (0.24396993068188416) + | > loader_time: 0.0116 (0.011933501481683432) + + + --> TIME: 2024-10-20 07:31:20 -- STEP: 2241/69059 -- GLOBAL_STEP: 71300 + | > loss_text_ce: 0.03157750889658928 (0.03260057033128973) + | > loss_mel_ce: 2.994687795639038 (3.108543186302644) + | > loss: 0.7565663456916809 (0.7852859393340677) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24357431618989658) + | > loader_time: 0.0099 (0.011923959243515676) + + + --> TIME: 2024-10-20 07:31:40 -- STEP: 2291/69059 -- GLOBAL_STEP: 71350 + | > loss_text_ce: 0.033156849443912506 (0.03260960921463037) + | > loss_mel_ce: 3.264888048171997 (3.1093286648945533) + | > loss: 0.8245112299919128 (0.7854845687740497) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.247 (0.24425577919642408) + | > loader_time: 0.0109 (0.01192852712936768) + + + --> TIME: 2024-10-20 07:31:58 -- STEP: 2341/69059 -- GLOBAL_STEP: 71400 + | > loss_text_ce: 0.03252185881137848 (0.03260423113273299) + | > loss_mel_ce: 3.099565267562866 (3.1084067068869916) + | > loss: 0.7830218076705933 (0.7852527347959445) + | > current_lr: 5e-06 + | > step_time: 0.2849 (0.24397234843358787) + | > loader_time: 0.0116 (0.01191627750127859) + + + --> TIME: 2024-10-20 07:32:17 -- STEP: 2391/69059 -- GLOBAL_STEP: 71450 + | > loss_text_ce: 0.03370120748877525 (0.03260435838531191) + | > loss_mel_ce: 3.418600559234619 (3.1092760297654807) + | > loss: 0.8630754351615906 (0.7854700972723787) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4914 (0.24429846218768048) + | > loader_time: 0.0138 (0.01190857915188269) + + + --> TIME: 2024-10-20 07:32:35 -- STEP: 2441/69059 -- GLOBAL_STEP: 71500 + | > loss_text_ce: 0.03140396252274513 (0.03260338836910776) + | > loss_mel_ce: 3.144425868988037 (3.1089403187823264) + | > loss: 0.7939574718475342 (0.7853859269145269) + | > current_lr: 5e-06 + | > step_time: 0.1677 (0.24402998383540397) + | > loader_time: 0.0105 (0.011901181137791338) + + + --> TIME: 2024-10-20 07:32:55 -- STEP: 2491/69059 -- GLOBAL_STEP: 71550 + | > loss_text_ce: 0.030057383701205254 (0.032606681695171245) + | > loss_mel_ce: 3.0535342693328857 (3.1093200571993562) + | > loss: 0.7708979249000549 (0.7854816848281299) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2376 (0.24440680216326383) + | > loader_time: 0.0111 (0.011900581063538957) + + + --> TIME: 2024-10-20 07:33:14 -- STEP: 2541/69059 -- GLOBAL_STEP: 71600 + | > loss_text_ce: 0.03374878689646721 (0.032599244335653796) + | > loss_mel_ce: 3.505150556564331 (3.109090931534719) + | > loss: 0.8847248554229736 (0.7854225440599556) + | > current_lr: 5e-06 + | > step_time: 0.2313 (0.24462063609967313) + | > loader_time: 0.0114 (0.011890268194445578) + + + --> TIME: 2024-10-20 07:33:32 -- STEP: 2591/69059 -- GLOBAL_STEP: 71650 + | > loss_text_ce: 0.03496154025197029 (0.032593997914972354) + | > loss_mel_ce: 3.0280306339263916 (3.1092079568212876) + | > loss: 0.7657480239868164 (0.7854504887545166) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3266 (0.2446193178847507) + | > loader_time: 0.0106 (0.011882640642808804) + + + --> TIME: 2024-10-20 07:33:49 -- STEP: 2641/69059 -- GLOBAL_STEP: 71700 + | > loss_text_ce: 0.03410597890615463 (0.03258678315236276) + | > loss_mel_ce: 2.8517913818359375 (3.1092036452901812) + | > loss: 0.7214743494987488 (0.7854476071631804) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24423557707235155) + | > loader_time: 0.0103 (0.01187396726568557) + + + --> TIME: 2024-10-20 07:34:08 -- STEP: 2691/69059 -- GLOBAL_STEP: 71750 + | > loss_text_ce: 0.030709194019436836 (0.032586765723024026) + | > loss_mel_ce: 2.9090051651000977 (3.1086303716430566) + | > loss: 0.7349286079406738 (0.7853042844147188) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.24434581975961697) + | > loader_time: 0.0113 (0.011875306749290743) + + + --> TIME: 2024-10-20 07:34:30 -- STEP: 2741/69059 -- GLOBAL_STEP: 71800 + | > loss_text_ce: 0.034644074738025665 (0.03259168039573802) + | > loss_mel_ce: 3.115511655807495 (3.1097964596809198) + | > loss: 0.7875389456748962 (0.7855970350680931) + | > current_lr: 5e-06 + | > step_time: 0.3389 (0.24511279741459108) + | > loader_time: 0.0112 (0.011876282488416932) + + + --> TIME: 2024-10-20 07:34:49 -- STEP: 2791/69059 -- GLOBAL_STEP: 71850 + | > loss_text_ce: 0.03002951107919216 (0.032583877349499975) + | > loss_mel_ce: 3.141718864440918 (3.1105153751988306) + | > loss: 0.7929370999336243 (0.7857748131861357) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4937 (0.2452181772876495) + | > loader_time: 0.013 (0.011876746344762849) + + + --> TIME: 2024-10-20 07:35:08 -- STEP: 2841/69059 -- GLOBAL_STEP: 71900 + | > loss_text_ce: 0.0343983992934227 (0.03258162877984306) + | > loss_mel_ce: 3.3832643032073975 (3.1103153437722155) + | > loss: 0.8544156551361084 (0.7857242431996448) + | > current_lr: 5e-06 + | > step_time: 0.5699 (0.24528284712351364) + | > loader_time: 0.0153 (0.011867996757612085) + + + --> TIME: 2024-10-20 07:35:26 -- STEP: 2891/69059 -- GLOBAL_STEP: 71950 + | > loss_text_ce: 0.0344475582242012 (0.03258289684374333) + | > loss_mel_ce: 3.2950713634490967 (3.110688197080832) + | > loss: 0.8323797583580017 (0.7858177735623254) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5952 (0.24534492919866455) + | > loader_time: 0.0154 (0.011860289238844338) + + + --> TIME: 2024-10-20 07:35:45 -- STEP: 2941/69059 -- GLOBAL_STEP: 72000 + | > loss_text_ce: 0.03278912231326103 (0.03258019866943057) + | > loss_mel_ce: 3.0160741806030273 (3.1108020952225344) + | > loss: 0.7622158527374268 (0.7858455735621344) + | > current_lr: 5e-06 + | > step_time: 0.3951 (0.24543582448664267) + | > loader_time: 0.0122 (0.011860387335825626) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_72000.pth + + --> TIME: 2024-10-20 07:36:19 -- STEP: 2991/69059 -- GLOBAL_STEP: 72050 + | > loss_text_ce: 0.029179345816373825 (0.03257643852653171) + | > loss_mel_ce: 3.175441265106201 (3.1103339082857513) + | > loss: 0.801155149936676 (0.7857275868052027) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.447 (0.24559527143352766) + | > loader_time: 0.0131 (0.011857838265595355) + + + --> TIME: 2024-10-20 07:36:38 -- STEP: 3041/69059 -- GLOBAL_STEP: 72100 + | > loss_text_ce: 0.031555112451314926 (0.032582236897348585) + | > loss_mel_ce: 3.3706095218658447 (3.1110125921620035) + | > loss: 0.8505411744117737 (0.785898707348906) + | > current_lr: 5e-06 + | > step_time: 0.2227 (0.245680070558452) + | > loader_time: 0.0109 (0.011863114056874194) + + + --> TIME: 2024-10-20 07:36:55 -- STEP: 3091/69059 -- GLOBAL_STEP: 72150 + | > loss_text_ce: 0.029326673597097397 (0.03257721948155073) + | > loss_mel_ce: 3.1171720027923584 (3.110677584133188) + | > loss: 0.7866246700286865 (0.7858137010139619) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.222 (0.24531407890023632) + | > loader_time: 0.0106 (0.011854490235722695) + + + --> TIME: 2024-10-20 07:37:14 -- STEP: 3141/69059 -- GLOBAL_STEP: 72200 + | > loss_text_ce: 0.03158168867230415 (0.0325754263836491) + | > loss_mel_ce: 2.716323137283325 (3.1104867344485507) + | > loss: 0.6869761943817139 (0.7857655402544539) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.2454039266404738) + | > loader_time: 0.0107 (0.011852761811794147) + + + --> TIME: 2024-10-20 07:37:33 -- STEP: 3191/69059 -- GLOBAL_STEP: 72250 + | > loss_text_ce: 0.03149763494729996 (0.03257185732872835) + | > loss_mel_ce: 3.1417737007141113 (3.111248173410334) + | > loss: 0.7933178544044495 (0.7859550077726161) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2061 (0.24538384078625639) + | > loader_time: 0.0106 (0.011847969208791788) + + + --> TIME: 2024-10-20 07:37:53 -- STEP: 3241/69059 -- GLOBAL_STEP: 72300 + | > loss_text_ce: 0.03242703527212143 (0.03257584859424816) + | > loss_mel_ce: 3.1646804809570312 (3.111068142449841) + | > loss: 0.7992768883705139 (0.7859109978144715) + | > current_lr: 5e-06 + | > step_time: 0.1845 (0.24574042352325054) + | > loader_time: 0.0107 (0.011851045902185131) + + + --> TIME: 2024-10-20 07:38:10 -- STEP: 3291/69059 -- GLOBAL_STEP: 72350 + | > loss_text_ce: 0.030554980039596558 (0.03257270975803052) + | > loss_mel_ce: 3.0831072330474854 (3.111305437708144) + | > loss: 0.7784155607223511 (0.7859695369043237) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.2455502341707149) + | > loader_time: 0.0103 (0.01184606204384986) + + + --> TIME: 2024-10-20 07:38:31 -- STEP: 3341/69059 -- GLOBAL_STEP: 72400 + | > loss_text_ce: 0.03180379420518875 (0.03257191796954059) + | > loss_mel_ce: 3.202216863632202 (3.111985053605068) + | > loss: 0.8085051774978638 (0.7861392429509373) + | > current_lr: 5e-06 + | > step_time: 0.1957 (0.2459679836643131) + | > loader_time: 0.0106 (0.011848166544827779) + + + --> TIME: 2024-10-20 07:38:49 -- STEP: 3391/69059 -- GLOBAL_STEP: 72450 + | > loss_text_ce: 0.03149109706282616 (0.03256307143593867) + | > loss_mel_ce: 2.889711856842041 (3.110843310908767) + | > loss: 0.7303007245063782 (0.785851595676673) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1968 (0.2459308037326175) + | > loader_time: 0.0109 (0.011844024585081554) + + + --> TIME: 2024-10-20 07:39:08 -- STEP: 3441/69059 -- GLOBAL_STEP: 72500 + | > loss_text_ce: 0.030307117849588394 (0.03256177606883353) + | > loss_mel_ce: 3.0353024005889893 (3.1106811773981375) + | > loss: 0.7664023637771606 (0.7858107384641791) + | > current_lr: 5e-06 + | > step_time: 0.2344 (0.24593911182045486) + | > loader_time: 0.0114 (0.011842896099251345) + + + --> TIME: 2024-10-20 07:39:28 -- STEP: 3491/69059 -- GLOBAL_STEP: 72550 + | > loss_text_ce: 0.03170076385140419 (0.032555515264579525) + | > loss_mel_ce: 3.3918721675872803 (3.111447953453924) + | > loss: 0.8558932542800903 (0.7860008673147498) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2816 (0.24630081841406798) + | > loader_time: 0.0119 (0.01184551089583754) + + + --> TIME: 2024-10-20 07:39:47 -- STEP: 3541/69059 -- GLOBAL_STEP: 72600 + | > loss_text_ce: 0.03366866707801819 (0.03255724936799459) + | > loss_mel_ce: 3.276862621307373 (3.111486408475492) + | > loss: 0.8276328444480896 (0.7860109146723334) + | > current_lr: 5e-06 + | > step_time: 0.3098 (0.24637951220674253) + | > loader_time: 0.012 (0.011848285217737642) + + + --> TIME: 2024-10-20 07:40:05 -- STEP: 3591/69059 -- GLOBAL_STEP: 72650 + | > loss_text_ce: 0.03321387618780136 (0.032555891301008846) + | > loss_mel_ce: 3.055253028869629 (3.11186510923678) + | > loss: 0.7721167206764221 (0.7861052504059864) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2039 (0.2463610623082687) + | > loader_time: 0.0109 (0.01184319022371724) + + + --> TIME: 2024-10-20 07:40:25 -- STEP: 3641/69059 -- GLOBAL_STEP: 72700 + | > loss_text_ce: 0.03015752322971821 (0.032555660036456754) + | > loss_mel_ce: 3.3815670013427734 (3.1121828353722556) + | > loss: 0.8529311418533325 (0.7861846241607865) + | > current_lr: 5e-06 + | > step_time: 0.2025 (0.24645970123739433) + | > loader_time: 0.0131 (0.011844450940931301) + + + --> TIME: 2024-10-20 07:40:42 -- STEP: 3691/69059 -- GLOBAL_STEP: 72750 + | > loss_text_ce: 0.03180868178606033 (0.03255453401537118) + | > loss_mel_ce: 3.077721118927002 (3.1115280812557233) + | > loss: 0.7773824334144592 (0.7860206540798114) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2039 (0.24631376203176808) + | > loader_time: 0.0109 (0.01183904053815432) + + + --> TIME: 2024-10-20 07:40:59 -- STEP: 3741/69059 -- GLOBAL_STEP: 72800 + | > loss_text_ce: 0.03182481974363327 (0.03255627835326278) + | > loss_mel_ce: 3.0361974239349365 (3.111251725069628) + | > loss: 0.7670055627822876 (0.7859520011329294) + | > current_lr: 5e-06 + | > step_time: 0.186 (0.2459789669727202) + | > loader_time: 0.0104 (0.011829893718394218) + + + --> TIME: 2024-10-20 07:41:17 -- STEP: 3791/69059 -- GLOBAL_STEP: 72850 + | > loss_text_ce: 0.03212899714708328 (0.03255245161118234) + | > loss_mel_ce: 2.888495683670044 (3.110933957581654) + | > loss: 0.7301561832427979 (0.7858716025889565) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1978 (0.24574700731364507) + | > loader_time: 0.0102 (0.011820162987212229) + + + --> TIME: 2024-10-20 07:41:36 -- STEP: 3841/69059 -- GLOBAL_STEP: 72900 + | > loss_text_ce: 0.03391551226377487 (0.032543736539802484) + | > loss_mel_ce: 2.824784994125366 (3.1111096404735297) + | > loss: 0.7146751284599304 (0.7859133445667246) + | > current_lr: 5e-06 + | > step_time: 0.2263 (0.24581425191834083) + | > loader_time: 0.0104 (0.011816646741784157) + + + --> TIME: 2024-10-20 07:41:54 -- STEP: 3891/69059 -- GLOBAL_STEP: 72950 + | > loss_text_ce: 0.034009456634521484 (0.03254191459433507) + | > loss_mel_ce: 3.7602427005767822 (3.11133917080838) + | > loss: 0.9485630393028259 (0.7859702716906805) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.6651 (0.24576870857734848) + | > loader_time: 0.016 (0.011811856346061763) + + + --> TIME: 2024-10-20 07:42:12 -- STEP: 3941/69059 -- GLOBAL_STEP: 73000 + | > loss_text_ce: 0.033213403075933456 (0.03254654494333291) + | > loss_mel_ce: 2.9660229682922363 (3.1114803401082614) + | > loss: 0.7498090863227844 (0.786006721619028) + | > current_lr: 5e-06 + | > step_time: 0.1769 (0.24566091640562432) + | > loader_time: 0.0108 (0.011805294613280434) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_73000.pth + + --> TIME: 2024-10-20 07:42:49 -- STEP: 3991/69059 -- GLOBAL_STEP: 73050 + | > loss_text_ce: 0.036903854459524155 (0.03254508412487678) + | > loss_mel_ce: 3.2975592613220215 (3.1118835786983357) + | > loss: 0.833615779876709 (0.7861071660297014) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3565 (0.2459150996312137) + | > loader_time: 0.0147 (0.011840497661849624) + + + --> TIME: 2024-10-20 07:43:07 -- STEP: 4041/69059 -- GLOBAL_STEP: 73100 + | > loss_text_ce: 0.03023240529000759 (0.032543254619341266) + | > loss_mel_ce: 3.158740758895874 (3.111769906367087) + | > loss: 0.7972432971000671 (0.7860782906077517) + | > current_lr: 5e-06 + | > step_time: 0.1865 (0.2457280542495905) + | > loader_time: 0.0104 (0.011834666033364636) + + + --> TIME: 2024-10-20 07:43:25 -- STEP: 4091/69059 -- GLOBAL_STEP: 73150 + | > loss_text_ce: 0.03328757733106613 (0.03253987118869523) + | > loss_mel_ce: 3.1518752574920654 (3.1121507094851077) + | > loss: 0.7962906956672668 (0.7861726455221081) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3865 (0.24571397921707425) + | > loader_time: 0.0132 (0.01183395145633525) + + + --> TIME: 2024-10-20 07:43:44 -- STEP: 4141/69059 -- GLOBAL_STEP: 73200 + | > loss_text_ce: 0.03267404064536095 (0.0325387779913845) + | > loss_mel_ce: 3.028480052947998 (3.1118278054907744) + | > loss: 0.7652885317802429 (0.7860916462159913) + | > current_lr: 5e-06 + | > step_time: 0.3745 (0.2458471911047369) + | > loader_time: 0.0129 (0.011834436498152369) + + + --> TIME: 2024-10-20 07:44:03 -- STEP: 4191/69059 -- GLOBAL_STEP: 73250 + | > loss_text_ce: 0.032634437084198 (0.03254188419337694) + | > loss_mel_ce: 3.262669801712036 (3.112281035307452) + | > loss: 0.8238260746002197 (0.7862057301856487) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4694 (0.2459383895476261) + | > loader_time: 0.0142 (0.011836488444319906) + + + --> TIME: 2024-10-20 07:44:22 -- STEP: 4241/69059 -- GLOBAL_STEP: 73300 + | > loss_text_ce: 0.036837249994277954 (0.03254115688589297) + | > loss_mel_ce: 2.9043221473693848 (3.111774268113223) + | > loss: 0.7352898716926575 (0.7860788565372357) + | > current_lr: 5e-06 + | > step_time: 0.4066 (0.24593764688293485) + | > loader_time: 0.0142 (0.01183489801973085) + + + --> TIME: 2024-10-20 07:44:41 -- STEP: 4291/69059 -- GLOBAL_STEP: 73350 + | > loss_text_ce: 0.03300517797470093 (0.032539642044583796) + | > loss_mel_ce: 2.97853422164917 (3.112179368671849) + | > loss: 0.7528848648071289 (0.786179752984377) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.199 (0.24595190083555507) + | > loader_time: 0.0105 (0.011832835762630238) + + + --> TIME: 2024-10-20 07:45:00 -- STEP: 4341/69059 -- GLOBAL_STEP: 73400 + | > loss_text_ce: 0.028233643621206284 (0.03253758686902106) + | > loss_mel_ce: 3.0601534843444824 (3.111884315286847) + | > loss: 0.7720967531204224 (0.7861054758134732) + | > current_lr: 5e-06 + | > step_time: 0.1723 (0.2459913843981381) + | > loader_time: 0.0106 (0.01183095480324185) + + + --> TIME: 2024-10-20 07:45:18 -- STEP: 4391/69059 -- GLOBAL_STEP: 73450 + | > loss_text_ce: 0.032459013164043427 (0.032537669511827974) + | > loss_mel_ce: 3.211461067199707 (3.1120630496999575) + | > loss: 0.8109800219535828 (0.7861501800663734) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.227 (0.24595274351854526) + | > loader_time: 0.0111 (0.01182550462007361) + + + --> TIME: 2024-10-20 07:45:35 -- STEP: 4441/69059 -- GLOBAL_STEP: 73500 + | > loss_text_ce: 0.03143307939171791 (0.03253279051470198) + | > loss_mel_ce: 3.304335355758667 (3.11171741347086) + | > loss: 0.8339421153068542 (0.7860625512668127) + | > current_lr: 5e-06 + | > step_time: 0.1842 (0.24558043748348368) + | > loader_time: 0.0107 (0.011817839520924063) + + + --> TIME: 2024-10-20 07:45:51 -- STEP: 4491/69059 -- GLOBAL_STEP: 73550 + | > loss_text_ce: 0.03242785483598709 (0.0325263850722712) + | > loss_mel_ce: 3.2233803272247314 (3.111138453172482) + | > loss: 0.8139520287513733 (0.785916209804855) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4109 (0.24525535826002456) + | > loader_time: 0.013 (0.011813903645931881) + + + --> TIME: 2024-10-20 07:46:09 -- STEP: 4541/69059 -- GLOBAL_STEP: 73600 + | > loss_text_ce: 0.03285517916083336 (0.03252645995344029) + | > loss_mel_ce: 2.7935290336608887 (3.1108067299767823) + | > loss: 0.7065960764884949 (0.7858332977292821) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.24514708495880375) + | > loader_time: 0.0103 (0.0118106916806058) + + + --> TIME: 2024-10-20 07:46:28 -- STEP: 4591/69059 -- GLOBAL_STEP: 73650 + | > loss_text_ce: 0.034671518951654434 (0.03252615199585199) + | > loss_mel_ce: 3.2240169048309326 (3.110659356315624) + | > loss: 0.8146721124649048 (0.7857963773347902) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4495 (0.24515696595993158) + | > loader_time: 0.0132 (0.011808300091014993) + + + --> TIME: 2024-10-20 07:46:47 -- STEP: 4641/69059 -- GLOBAL_STEP: 73700 + | > loss_text_ce: 0.03207595646381378 (0.032529428197912644) + | > loss_mel_ce: 3.303075075149536 (3.110999411484515) + | > loss: 0.833787739276886 (0.7858822101972355) + | > current_lr: 5e-06 + | > step_time: 0.3356 (0.2453239333960959) + | > loader_time: 0.0139 (0.011810787380114141) + + + --> TIME: 2024-10-20 07:47:06 -- STEP: 4691/69059 -- GLOBAL_STEP: 73750 + | > loss_text_ce: 0.035563111305236816 (0.03253002549029067) + | > loss_mel_ce: 3.402392625808716 (3.1109465030535173) + | > loss: 0.8594889640808105 (0.7858691323771714) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.303 (0.24531911065445808) + | > loader_time: 0.0123 (0.011806022787978486) + + + --> TIME: 2024-10-20 07:47:24 -- STEP: 4741/69059 -- GLOBAL_STEP: 73800 + | > loss_text_ce: 0.0326971597969532 (0.03252950402758934) + | > loss_mel_ce: 2.8162271976470947 (3.1105862649476266) + | > loss: 0.7122310996055603 (0.7857789425149899) + | > current_lr: 5e-06 + | > step_time: 0.197 (0.24532525073222233) + | > loader_time: 0.0104 (0.01180067285659621) + + + --> TIME: 2024-10-20 07:47:42 -- STEP: 4791/69059 -- GLOBAL_STEP: 73850 + | > loss_text_ce: 0.030108148232102394 (0.03252717381818727) + | > loss_mel_ce: 3.152071952819824 (3.1105023588922216) + | > loss: 0.7955450415611267 (0.7857573834821147) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.24525127861749102) + | > loader_time: 0.011 (0.011794491914985241) + + + --> TIME: 2024-10-20 07:48:01 -- STEP: 4841/69059 -- GLOBAL_STEP: 73900 + | > loss_text_ce: 0.03363398462533951 (0.03252668663292997) + | > loss_mel_ce: 3.2961933612823486 (3.110846469974105) + | > loss: 0.8324568271636963 (0.7858432894820792) + | > current_lr: 5e-06 + | > step_time: 0.4846 (0.24527071103949213) + | > loader_time: 0.0137 (0.011793088184065931) + + + --> TIME: 2024-10-20 07:48:20 -- STEP: 4891/69059 -- GLOBAL_STEP: 73950 + | > loss_text_ce: 0.034139566123485565 (0.03252835988471345) + | > loss_mel_ce: 3.235874891281128 (3.1104500232311243) + | > loss: 0.8175036311149597 (0.7857445961028565) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2468 (0.24528171045603234) + | > loader_time: 0.0109 (0.011790204111527055) + + + --> TIME: 2024-10-20 07:48:38 -- STEP: 4941/69059 -- GLOBAL_STEP: 74000 + | > loss_text_ce: 0.033997807651758194 (0.03252960286583646) + | > loss_mel_ce: 3.039440631866455 (3.110384633970079) + | > loss: 0.7683596014976501 (0.7857285594679618) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.24516331857574344) + | > loader_time: 0.0106 (0.011784327896325091) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_74000.pth + + --> TIME: 2024-10-20 07:49:11 -- STEP: 4991/69059 -- GLOBAL_STEP: 74050 + | > loss_text_ce: 0.031319499015808105 (0.0325302035967457) + | > loss_mel_ce: 3.188300132751465 (3.1100494162732057) + | > loss: 0.8049049377441406 (0.7856449052151059) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2037 (0.2450253701692496) + | > loader_time: 0.0104 (0.011779791175232382) + + + --> TIME: 2024-10-20 07:49:29 -- STEP: 5041/69059 -- GLOBAL_STEP: 74100 + | > loss_text_ce: 0.03616201505064964 (0.03253107209943206) + | > loss_mel_ce: 3.3670387268066406 (3.1100847715424362) + | > loss: 0.8508001565933228 (0.7856539611399254) + | > current_lr: 5e-06 + | > step_time: 0.54 (0.24505946830586056) + | > loader_time: 0.0155 (0.011777276960636664) + + + --> TIME: 2024-10-20 07:49:48 -- STEP: 5091/69059 -- GLOBAL_STEP: 74150 + | > loss_text_ce: 0.029410693794488907 (0.03252891707412446) + | > loss_mel_ce: 2.88852596282959 (3.1100436673139087) + | > loss: 0.7294841408729553 (0.7856431462586307) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.195 (0.2450310442588625) + | > loader_time: 0.0103 (0.011773364292430274) + + + --> TIME: 2024-10-20 07:50:08 -- STEP: 5141/69059 -- GLOBAL_STEP: 74200 + | > loss_text_ce: 0.033876463770866394 (0.03252717220322086) + | > loss_mel_ce: 3.3415799140930176 (3.1102062027131145) + | > loss: 0.8438640832901001 (0.7856833438677581) + | > current_lr: 5e-06 + | > step_time: 0.5736 (0.24523748119757163) + | > loader_time: 0.0174 (0.011775500299957947) + + + --> TIME: 2024-10-20 07:50:26 -- STEP: 5191/69059 -- GLOBAL_STEP: 74250 + | > loss_text_ce: 0.03233807161450386 (0.03252763495364476) + | > loss_mel_ce: 3.044264078140259 (3.1100563902865033) + | > loss: 0.7691505551338196 (0.7856460064391225) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3953 (0.2452501635679136) + | > loader_time: 0.011 (0.011773996696699991) + + + --> TIME: 2024-10-20 07:50:45 -- STEP: 5241/69059 -- GLOBAL_STEP: 74300 + | > loss_text_ce: 0.03381272405385971 (0.032529604440866666) + | > loss_mel_ce: 3.1677050590515137 (3.109929470314458) + | > loss: 0.8003794550895691 (0.7856147688232604) + | > current_lr: 5e-06 + | > step_time: 0.4895 (0.24529854051713337) + | > loader_time: 0.0157 (0.011775151074239727) + + + --> TIME: 2024-10-20 07:51:04 -- STEP: 5291/69059 -- GLOBAL_STEP: 74350 + | > loss_text_ce: 0.03500755876302719 (0.0325263790688399) + | > loss_mel_ce: 3.2998342514038086 (3.109874609895114) + | > loss: 0.8337104320526123 (0.785600247353377) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1996 (0.24529560854610255) + | > loader_time: 0.014 (0.011776159760902896) + + + --> TIME: 2024-10-20 07:51:22 -- STEP: 5341/69059 -- GLOBAL_STEP: 74400 + | > loss_text_ce: 0.03295676037669182 (0.03252268707537678) + | > loss_mel_ce: 3.0201711654663086 (3.110290645459376) + | > loss: 0.7632820010185242 (0.7857033332687393) + | > current_lr: 5e-06 + | > step_time: 0.2308 (0.2453089953584855) + | > loader_time: 0.0121 (0.011776147498505963) + + + --> TIME: 2024-10-20 07:51:39 -- STEP: 5391/69059 -- GLOBAL_STEP: 74450 + | > loss_text_ce: 0.03036847710609436 (0.032521969178027056) + | > loss_mel_ce: 2.9657254219055176 (3.109734198087842) + | > loss: 0.7490234971046448 (0.7855640419381729) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1849 (0.24509901298129522) + | > loader_time: 0.0103 (0.011769921588137054) + + + --> TIME: 2024-10-20 07:51:59 -- STEP: 5441/69059 -- GLOBAL_STEP: 74500 + | > loss_text_ce: 0.0327354334294796 (0.032523923393142404) + | > loss_mel_ce: 2.9847707748413086 (3.1097393760653107) + | > loss: 0.7543765306472778 (0.7855658249652595) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.2452482491419434) + | > loader_time: 0.013 (0.011772885565327466) + + + --> TIME: 2024-10-20 07:52:19 -- STEP: 5491/69059 -- GLOBAL_STEP: 74550 + | > loss_text_ce: 0.03357201814651489 (0.03252098477760846) + | > loss_mel_ce: 3.134495973587036 (3.1101425525986244) + | > loss: 0.7920169830322266 (0.7856658844321697) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.24536777762762293) + | > loader_time: 0.0103 (0.01177236430741817) + + + --> TIME: 2024-10-20 07:52:36 -- STEP: 5541/69059 -- GLOBAL_STEP: 74600 + | > loss_text_ce: 0.03193668648600578 (0.03251759013629329) + | > loss_mel_ce: 3.0984275341033936 (3.1101178377248218) + | > loss: 0.7825910449028015 (0.7856588570182231) + | > current_lr: 5e-06 + | > step_time: 0.2327 (0.2452083767762018) + | > loader_time: 0.0121 (0.011768027475822118) + + + --> TIME: 2024-10-20 07:52:54 -- STEP: 5591/69059 -- GLOBAL_STEP: 74650 + | > loss_text_ce: 0.03420315682888031 (0.032518130109617606) + | > loss_mel_ce: 3.22890305519104 (3.110312086080443) + | > loss: 0.815776526927948 (0.7857075541161439) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2178 (0.2451879668163285) + | > loader_time: 0.0142 (0.01176428658402345) + + + --> TIME: 2024-10-20 07:53:13 -- STEP: 5641/69059 -- GLOBAL_STEP: 74700 + | > loss_text_ce: 0.029795851558446884 (0.032515952578238756) + | > loss_mel_ce: 3.1542654037475586 (3.1099586775911083) + | > loss: 0.7960153222084045 (0.7856186576038358) + | > current_lr: 5e-06 + | > step_time: 0.176 (0.2451543113987109) + | > loader_time: 0.0104 (0.011759657742304716) + + + --> TIME: 2024-10-20 07:53:31 -- STEP: 5691/69059 -- GLOBAL_STEP: 74750 + | > loss_text_ce: 0.033887650817632675 (0.0325174386418493) + | > loss_mel_ce: 3.145369052886963 (3.109991568795115) + | > loss: 0.7948141694068909 (0.7856272518938524) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3904 (0.245164706698203) + | > loader_time: 0.0122 (0.011757393458086055) + + + --> TIME: 2024-10-20 07:53:50 -- STEP: 5741/69059 -- GLOBAL_STEP: 74800 + | > loss_text_ce: 0.03222284093499184 (0.032515391608464256) + | > loss_mel_ce: 3.3645780086517334 (3.109806908192358) + | > loss: 0.8492001891136169 (0.7855805749470419) + | > current_lr: 5e-06 + | > step_time: 0.2161 (0.2451594622993569) + | > loader_time: 0.0112 (0.011754683803710527) + + + --> TIME: 2024-10-20 07:54:09 -- STEP: 5791/69059 -- GLOBAL_STEP: 74850 + | > loss_text_ce: 0.028975244611501694 (0.03251692850192246) + | > loss_mel_ce: 3.0500638484954834 (3.1096796588028184) + | > loss: 0.7697597742080688 (0.7855491468355126) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1951 (0.24521848501226196) + | > loader_time: 0.0103 (0.011753812311350685) + + + --> TIME: 2024-10-20 07:54:28 -- STEP: 5841/69059 -- GLOBAL_STEP: 74900 + | > loss_text_ce: 0.030262261629104614 (0.03251598800228452) + | > loss_mel_ce: 2.929910182952881 (3.1095341326255257) + | > loss: 0.7400431036949158 (0.7855125301465086) + | > current_lr: 5e-06 + | > step_time: 0.1953 (0.24533417612247896) + | > loader_time: 0.0135 (0.01175483761738271) + + + --> TIME: 2024-10-20 07:54:47 -- STEP: 5891/69059 -- GLOBAL_STEP: 74950 + | > loss_text_ce: 0.03356323018670082 (0.03251526796372334) + | > loss_mel_ce: 3.115647554397583 (3.109619946658682) + | > loss: 0.7873026728630066 (0.7855338036578934) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2596 (0.24539565408681088) + | > loader_time: 0.0106 (0.011753969697591278) + + + --> TIME: 2024-10-20 07:55:04 -- STEP: 5941/69059 -- GLOBAL_STEP: 75000 + | > loss_text_ce: 0.031982969492673874 (0.032510062762418655) + | > loss_mel_ce: 2.9521324634552 (3.109238723816845) + | > loss: 0.7460288405418396 (0.7854371966285908) + | > current_lr: 5e-06 + | > step_time: 0.1937 (0.2451848413101091) + | > loader_time: 0.0106 (0.01174893328646302) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_75000.pth + + --> TIME: 2024-10-20 07:55:43 -- STEP: 5991/69059 -- GLOBAL_STEP: 75050 + | > loss_text_ce: 0.033609744161367416 (0.0325115389345912) + | > loss_mel_ce: 3.1053197383880615 (3.1094759583493103) + | > loss: 0.7847323417663574 (0.7854968742981234) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2884 (0.2453428152864673) + | > loader_time: 0.0115 (0.011750512560069367) + + + --> TIME: 2024-10-20 07:56:02 -- STEP: 6041/69059 -- GLOBAL_STEP: 75100 + | > loss_text_ce: 0.030867649242281914 (0.03251066571849252) + | > loss_mel_ce: 2.703324556350708 (3.109569719385931) + | > loss: 0.6835480332374573 (0.7855200962581451) + | > current_lr: 5e-06 + | > step_time: 0.201 (0.2454894286002412) + | > loader_time: 0.0109 (0.011752011937389442) + + + --> TIME: 2024-10-20 07:56:19 -- STEP: 6091/69059 -- GLOBAL_STEP: 75150 + | > loss_text_ce: 0.030739080160856247 (0.03250949216559782) + | > loss_mel_ce: 2.77230167388916 (3.1092506087564167) + | > loss: 0.7007601857185364 (0.7854400252477811) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.24531293991770264) + | > loader_time: 0.0106 (0.011746496309534993) + + + --> TIME: 2024-10-20 07:56:38 -- STEP: 6141/69059 -- GLOBAL_STEP: 75200 + | > loss_text_ce: 0.035023149102926254 (0.032511372796789485) + | > loss_mel_ce: 3.7135658264160156 (3.109745646237434) + | > loss: 0.9371472597122192 (0.7855642547515794) + | > current_lr: 5e-06 + | > step_time: 0.3452 (0.24530336822288623) + | > loader_time: 0.0133 (0.01174463520724036) + + + --> TIME: 2024-10-20 07:56:57 -- STEP: 6191/69059 -- GLOBAL_STEP: 75250 + | > loss_text_ce: 0.03294851630926132 (0.03251459274517419) + | > loss_mel_ce: 3.2569072246551514 (3.1101234353160536) + | > loss: 0.8224639296531677 (0.7856595070253103) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4666 (0.2453866106611599) + | > loader_time: 0.0137 (0.011744652208806716) + + + --> TIME: 2024-10-20 07:57:14 -- STEP: 6241/69059 -- GLOBAL_STEP: 75300 + | > loss_text_ce: 0.03388669341802597 (0.03251314396359482) + | > loss_mel_ce: 2.9492342472076416 (3.1097477713641286) + | > loss: 0.7457802295684814 (0.7855652288411826) + | > current_lr: 5e-06 + | > step_time: 0.3478 (0.2452415889739379) + | > loader_time: 0.0122 (0.011742652751871889) + + + --> TIME: 2024-10-20 07:57:33 -- STEP: 6291/69059 -- GLOBAL_STEP: 75350 + | > loss_text_ce: 0.03238147124648094 (0.032510185125990756) + | > loss_mel_ce: 3.2446281909942627 (3.109491977189876) + | > loss: 0.8192524313926697 (0.7855005406116834) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.421 (0.24526537035061471) + | > loader_time: 0.0128 (0.011741660013048789) + + + --> TIME: 2024-10-20 07:57:51 -- STEP: 6341/69059 -- GLOBAL_STEP: 75400 + | > loss_text_ce: 0.031889088451862335 (0.03250788302292837) + | > loss_mel_ce: 2.8268043994903564 (3.109510013760499) + | > loss: 0.7146733999252319 (0.7855044742372014) + | > current_lr: 5e-06 + | > step_time: 0.2319 (0.24525426408316206) + | > loader_time: 0.0109 (0.01173956505866143) + + + --> TIME: 2024-10-20 07:58:12 -- STEP: 6391/69059 -- GLOBAL_STEP: 75450 + | > loss_text_ce: 0.035325173288583755 (0.032505024297673206) + | > loss_mel_ce: 3.478933334350586 (3.1095897578014267) + | > loss: 0.8785646557807922 (0.7855236955491834) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3911 (0.24544352411459056) + | > loader_time: 0.0124 (0.011741966246961721) + + + --> TIME: 2024-10-20 07:58:30 -- STEP: 6441/69059 -- GLOBAL_STEP: 75500 + | > loss_text_ce: 0.031148754060268402 (0.03250323629194426) + | > loss_mel_ce: 2.700897216796875 (3.109499703281286) + | > loss: 0.6830114722251892 (0.7855007349195509) + | > current_lr: 5e-06 + | > step_time: 0.1787 (0.24542536840659457) + | > loader_time: 0.0113 (0.011742775271054006) + + + --> TIME: 2024-10-20 07:58:48 -- STEP: 6491/69059 -- GLOBAL_STEP: 75550 + | > loss_text_ce: 0.03094315156340599 (0.03250247600084188) + | > loss_mel_ce: 2.9507906436920166 (3.1094038919902256) + | > loss: 0.7454334497451782 (0.7854765919999187) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2143 (0.24535944277802008) + | > loader_time: 0.0107 (0.01174063286599863) + + + --> TIME: 2024-10-20 07:59:06 -- STEP: 6541/69059 -- GLOBAL_STEP: 75600 + | > loss_text_ce: 0.031100604683160782 (0.0325023961466896) + | > loss_mel_ce: 3.1399526596069336 (3.1096085962617166) + | > loss: 0.7927632927894592 (0.7855277481273741) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.24529486314143414) + | > loader_time: 0.012 (0.011739016053071915) + + + --> TIME: 2024-10-20 07:59:25 -- STEP: 6591/69059 -- GLOBAL_STEP: 75650 + | > loss_text_ce: 0.03256784752011299 (0.03249886982112412) + | > loss_mel_ce: 3.3933358192443848 (3.1094312600925975) + | > loss: 0.8564758896827698 (0.7854825325085274) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4984 (0.2453295471562979) + | > loader_time: 0.0132 (0.011735878083326457) + + + --> TIME: 2024-10-20 07:59:45 -- STEP: 6641/69059 -- GLOBAL_STEP: 75700 + | > loss_text_ce: 0.03274547681212425 (0.032498143373250604) + | > loss_mel_ce: 3.0823793411254883 (3.109557165113707) + | > loss: 0.7787811756134033 (0.7855138271136052) + | > current_lr: 5e-06 + | > step_time: 0.2054 (0.24543011897551945) + | > loader_time: 0.0109 (0.011734382195423186) + + + --> TIME: 2024-10-20 08:00:03 -- STEP: 6691/69059 -- GLOBAL_STEP: 75750 + | > loss_text_ce: 0.030623845756053925 (0.032496965662181757) + | > loss_mel_ce: 3.4887430667877197 (3.109504470873656) + | > loss: 0.8798417448997498 (0.7855003591414754) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3581 (0.24538793034340445) + | > loader_time: 0.0151 (0.011731218212015384) + + + --> TIME: 2024-10-20 08:00:20 -- STEP: 6741/69059 -- GLOBAL_STEP: 75800 + | > loss_text_ce: 0.03326696902513504 (0.032494894508092884) + | > loss_mel_ce: 2.828530788421631 (3.109226922857321) + | > loss: 0.7154494524002075 (0.785430454328919) + | > current_lr: 5e-06 + | > step_time: 0.2026 (0.24527935878474677) + | > loader_time: 0.0113 (0.011730008089229391) + + + --> TIME: 2024-10-20 08:00:38 -- STEP: 6791/69059 -- GLOBAL_STEP: 75850 + | > loss_text_ce: 0.030605902895331383 (0.03249253667375382) + | > loss_mel_ce: 2.628248929977417 (3.1094298295675613) + | > loss: 0.6647136807441711 (0.7854805915314603) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2267 (0.24522662232996806) + | > loader_time: 0.0111 (0.011726248862904536) + + + --> TIME: 2024-10-20 08:00:55 -- STEP: 6841/69059 -- GLOBAL_STEP: 75900 + | > loss_text_ce: 0.02853454276919365 (0.0324894561021301) + | > loss_mel_ce: 3.0148308277130127 (3.109095680488862) + | > loss: 0.7608413696289062 (0.7853962841254891) + | > current_lr: 5e-06 + | > step_time: 0.1935 (0.24506841384080258) + | > loader_time: 0.0103 (0.011722458470130847) + + + --> TIME: 2024-10-20 08:01:15 -- STEP: 6891/69059 -- GLOBAL_STEP: 75950 + | > loss_text_ce: 0.03339548408985138 (0.032488354108606024) + | > loss_mel_ce: 3.0633275508880615 (3.1094353424587644) + | > loss: 0.7741807699203491 (0.7854809241117714) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1757 (0.2451309126739214) + | > loader_time: 0.0098 (0.011741685929586965) + + + --> TIME: 2024-10-20 08:01:34 -- STEP: 6941/69059 -- GLOBAL_STEP: 76000 + | > loss_text_ce: 0.031563423573970795 (0.032487825142076696) + | > loss_mel_ce: 3.203937530517578 (3.109403899897246) + | > loss: 0.8088752627372742 (0.785472931220919) + | > current_lr: 5e-06 + | > step_time: 0.1851 (0.24516011067588156) + | > loader_time: 0.0104 (0.011739837867419166) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_76000.pth + + --> TIME: 2024-10-20 08:02:09 -- STEP: 6991/69059 -- GLOBAL_STEP: 76050 + | > loss_text_ce: 0.032634735107421875 (0.03248979685397271) + | > loss_mel_ce: 3.714634895324707 (3.109514346366242) + | > loss: 0.9368174076080322 (0.7855010357661538) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.497 (0.24542036057882702) + | > loader_time: 0.0182 (0.01174206693461314) + + + --> TIME: 2024-10-20 08:02:27 -- STEP: 7041/69059 -- GLOBAL_STEP: 76100 + | > loss_text_ce: 0.033249884843826294 (0.0324897070450635) + | > loss_mel_ce: 3.487938404083252 (3.109284603600813) + | > loss: 0.880297064781189 (0.7854435776313109) + | > current_lr: 5e-06 + | > step_time: 0.2317 (0.24534643117955862) + | > loader_time: 0.0112 (0.011741268788865823) + + + --> TIME: 2024-10-20 08:02:46 -- STEP: 7091/69059 -- GLOBAL_STEP: 76150 + | > loss_text_ce: 0.03515750169754028 (0.03248955184225264) + | > loss_mel_ce: 3.4785451889038086 (3.109207741213255) + | > loss: 0.878425657749176 (0.785424323264205) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4121 (0.2453459035423945) + | > loader_time: 0.0132 (0.011739518541221216) + + + --> TIME: 2024-10-20 08:03:04 -- STEP: 7141/69059 -- GLOBAL_STEP: 76200 + | > loss_text_ce: 0.030498748645186424 (0.032488535502251246) + | > loss_mel_ce: 3.162663221359253 (3.109115105953358) + | > loss: 0.798290491104126 (0.7854009103314278) + | > current_lr: 5e-06 + | > step_time: 0.2162 (0.24528956373204305) + | > loader_time: 0.0108 (0.011737327217771945) + + + --> TIME: 2024-10-20 08:03:22 -- STEP: 7191/69059 -- GLOBAL_STEP: 76250 + | > loss_text_ce: 0.030374685302376747 (0.0324855510771229) + | > loss_mel_ce: 3.033112049102783 (3.1091767162760116) + | > loss: 0.7658717036247253 (0.7854155667997534) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2074 (0.2452241773980663) + | > loader_time: 0.0131 (0.011735732358779463) + + + --> TIME: 2024-10-20 08:03:40 -- STEP: 7241/69059 -- GLOBAL_STEP: 76300 + | > loss_text_ce: 0.030393676832318306 (0.03248576046362625) + | > loss_mel_ce: 3.3496527671813965 (3.109204048478888) + | > loss: 0.8450115919113159 (0.7854224521882326) + | > current_lr: 5e-06 + | > step_time: 0.2311 (0.24517467834889897) + | > loader_time: 0.0113 (0.011736431300944686) + + + --> TIME: 2024-10-20 08:03:57 -- STEP: 7291/69059 -- GLOBAL_STEP: 76350 + | > loss_text_ce: 0.03286769986152649 (0.03247949967198887) + | > loss_mel_ce: 3.0276505947113037 (3.1091062221780854) + | > loss: 0.765129566192627 (0.7853964304109768) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.24501414953673137) + | > loader_time: 0.0107 (0.01173286607376328) + + + --> TIME: 2024-10-20 08:04:17 -- STEP: 7341/69059 -- GLOBAL_STEP: 76400 + | > loss_text_ce: 0.032883044332265854 (0.03248110192576308) + | > loss_mel_ce: 3.190746307373047 (3.109343508221671) + | > loss: 0.8059073090553284 (0.7854561524874439) + | > current_lr: 5e-06 + | > step_time: 0.4054 (0.24511198629850328) + | > loader_time: 0.0131 (0.011733571964394358) + + + --> TIME: 2024-10-20 08:04:34 -- STEP: 7391/69059 -- GLOBAL_STEP: 76450 + | > loss_text_ce: 0.032763805240392685 (0.032479754773220265) + | > loss_mel_ce: 3.1760482788085938 (3.108874381241755) + | > loss: 0.8022029995918274 (0.7853385339633582) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3873 (0.24503895410938492) + | > loader_time: 0.0121 (0.011731728151321681) + + + --> TIME: 2024-10-20 08:04:52 -- STEP: 7441/69059 -- GLOBAL_STEP: 76500 + | > loss_text_ce: 0.032928235828876495 (0.03247549395045519) + | > loss_mel_ce: 2.858797788619995 (3.1093006722309657) + | > loss: 0.7229315042495728 (0.7854440414910352) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.24500083493987745) + | > loader_time: 0.011 (0.011731295091557011) + + + --> TIME: 2024-10-20 08:05:11 -- STEP: 7491/69059 -- GLOBAL_STEP: 76550 + | > loss_text_ce: 0.031421687453985214 (0.03247540488445292) + | > loss_mel_ce: 2.855015277862549 (3.108867701884504) + | > loss: 0.7216092348098755 (0.7853357766359195) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2091 (0.24501739240841655) + | > loader_time: 0.0108 (0.011732147653722564) + + + --> TIME: 2024-10-20 08:05:31 -- STEP: 7541/69059 -- GLOBAL_STEP: 76600 + | > loss_text_ce: 0.031771451234817505 (0.03247531461300345) + | > loss_mel_ce: 2.821772336959839 (3.1089989439654833) + | > loss: 0.7133859395980835 (0.7853685645980615) + | > current_lr: 5e-06 + | > step_time: 0.2237 (0.2451194505637148) + | > loader_time: 0.0111 (0.011732218800165228) + + + --> TIME: 2024-10-20 08:05:48 -- STEP: 7591/69059 -- GLOBAL_STEP: 76650 + | > loss_text_ce: 0.03071879781782627 (0.032470362679548254) + | > loss_mel_ce: 3.007721424102783 (3.1086769984999973) + | > loss: 0.7596100568771362 (0.7852868402511479) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1973 (0.24499057275538166) + | > loader_time: 0.0107 (0.011730222170602014) + + + --> TIME: 2024-10-20 08:06:06 -- STEP: 7641/69059 -- GLOBAL_STEP: 76700 + | > loss_text_ce: 0.03345347195863724 (0.032467603222015466) + | > loss_mel_ce: 3.0413999557495117 (3.108548810759416) + | > loss: 0.7687133550643921 (0.7852541034235064) + | > current_lr: 5e-06 + | > step_time: 0.1762 (0.2449492389440318) + | > loader_time: 0.0105 (0.011726947436702776) + + + --> TIME: 2024-10-20 08:06:25 -- STEP: 7691/69059 -- GLOBAL_STEP: 76750 + | > loss_text_ce: 0.030327240005135536 (0.03246612631306669) + | > loss_mel_ce: 3.2596371173858643 (3.108314191081437) + | > loss: 0.8224911093711853 (0.785195079260228) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3036 (0.24504723826026345) + | > loader_time: 0.0114 (0.011724783995730915) + + + --> TIME: 2024-10-20 08:06:43 -- STEP: 7741/69059 -- GLOBAL_STEP: 76800 + | > loss_text_ce: 0.0333230122923851 (0.03246360943765667) + | > loss_mel_ce: 3.2818593978881836 (3.1081433525967546) + | > loss: 0.8287956118583679 (0.7851517404362359) + | > current_lr: 5e-06 + | > step_time: 0.3797 (0.2449570545437508) + | > loader_time: 0.0151 (0.011721447785839136) + + + --> TIME: 2024-10-20 08:07:02 -- STEP: 7791/69059 -- GLOBAL_STEP: 76850 + | > loss_text_ce: 0.03150520846247673 (0.03246031748878037) + | > loss_mel_ce: 3.532609701156616 (3.1081349304376955) + | > loss: 0.8910287022590637 (0.7851488119125853) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4162 (0.24496260489261956) + | > loader_time: 0.0149 (0.01171902838452242) + + + --> TIME: 2024-10-20 08:07:21 -- STEP: 7841/69059 -- GLOBAL_STEP: 76900 + | > loss_text_ce: 0.03221982717514038 (0.03246187339523191) + | > loss_mel_ce: 2.9788339138031006 (3.1081907348720383) + | > loss: 0.7527634501457214 (0.7851631520293435) + | > current_lr: 5e-06 + | > step_time: 0.2067 (0.24499535396411976) + | > loader_time: 0.0108 (0.011715711767067498) + + + --> TIME: 2024-10-20 08:07:40 -- STEP: 7891/69059 -- GLOBAL_STEP: 76950 + | > loss_text_ce: 0.03186240419745445 (0.0324599169494006) + | > loss_mel_ce: 3.309811592102051 (3.1080201304226285) + | > loss: 0.8354185223579407 (0.7851200118035873) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5356 (0.24514118740728125) + | > loader_time: 0.0133 (0.011715072323169847) + + + --> TIME: 2024-10-20 08:07:58 -- STEP: 7941/69059 -- GLOBAL_STEP: 77000 + | > loss_text_ce: 0.0306599959731102 (0.03245974012505477) + | > loss_mel_ce: 3.2034049034118652 (3.1077401650040506) + | > loss: 0.8085162043571472 (0.785049976261928) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.24501811636888834) + | > loader_time: 0.0105 (0.011710842885804148) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_77000.pth + + --> TIME: 2024-10-20 08:08:30 -- STEP: 7991/69059 -- GLOBAL_STEP: 77050 + | > loss_text_ce: 0.032094087451696396 (0.03245856477097547) + | > loss_mel_ce: 2.815509557723999 (3.107615159160875) + | > loss: 0.7119008898735046 (0.7850184309484064) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2834 (0.24492464395536676) + | > loader_time: 0.0111 (0.011709408383541908) + + + --> TIME: 2024-10-20 08:08:49 -- STEP: 8041/69059 -- GLOBAL_STEP: 77100 + | > loss_text_ce: 0.03267469257116318 (0.03245622124636433) + | > loss_mel_ce: 2.968280076980591 (3.1076831701397527) + | > loss: 0.7502387166023254 (0.7850348477991579) + | > current_lr: 5e-06 + | > step_time: 0.2351 (0.24491603928053501) + | > loader_time: 0.0112 (0.01170817813296889) + + + --> TIME: 2024-10-20 08:09:07 -- STEP: 8091/69059 -- GLOBAL_STEP: 77150 + | > loss_text_ce: 0.030590655282139778 (0.032455909433435695) + | > loss_mel_ce: 2.8556604385375977 (3.1076225016559396) + | > loss: 0.7215628027915955 (0.785019602738445) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2042 (0.24488449043638846) + | > loader_time: 0.0104 (0.011707200504265743) + + + --> TIME: 2024-10-20 08:09:24 -- STEP: 8141/69059 -- GLOBAL_STEP: 77200 + | > loss_text_ce: 0.03361831232905388 (0.0324528695290874) + | > loss_mel_ce: 3.011624574661255 (3.107205808960737) + | > loss: 0.7613106966018677 (0.7849146695925405) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.2447468880396952) + | > loader_time: 0.0105 (0.011705183130622052) + + + --> TIME: 2024-10-20 08:09:42 -- STEP: 8191/69059 -- GLOBAL_STEP: 77250 + | > loss_text_ce: 0.030235713347792625 (0.03245384948758612) + | > loss_mel_ce: 3.1010022163391113 (3.107212603594296) + | > loss: 0.7828094959259033 (0.7849166132332333) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2336 (0.2447365837763145) + | > loader_time: 0.0118 (0.011703802252438705) + + + --> TIME: 2024-10-20 08:10:02 -- STEP: 8241/69059 -- GLOBAL_STEP: 77300 + | > loss_text_ce: 0.033178362995386124 (0.03245523038217746) + | > loss_mel_ce: 3.342205047607422 (3.107322000152436) + | > loss: 0.8438458442687988 (0.7849443076001454) + | > current_lr: 5e-06 + | > step_time: 0.3663 (0.2448100691613548) + | > loader_time: 0.017 (0.011704677806984332) + + + --> TIME: 2024-10-20 08:10:20 -- STEP: 8291/69059 -- GLOBAL_STEP: 77350 + | > loss_text_ce: 0.03275788575410843 (0.03245133589250007) + | > loss_mel_ce: 3.408498525619507 (3.107205719729842) + | > loss: 0.860314130783081 (0.7849142638841304) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1997 (0.24476602591934546) + | > loader_time: 0.0162 (0.011702512058529067) + + + --> TIME: 2024-10-20 08:10:38 -- STEP: 8341/69059 -- GLOBAL_STEP: 77400 + | > loss_text_ce: 0.03154725581407547 (0.03244958432723356) + | > loss_mel_ce: 3.2368414402008057 (3.106866329040134) + | > loss: 0.8170971870422363 (0.7848289783031527) + | > current_lr: 5e-06 + | > step_time: 0.28 (0.2447447375761864) + | > loader_time: 0.0118 (0.011699976301953581) + + + --> TIME: 2024-10-20 08:10:58 -- STEP: 8391/69059 -- GLOBAL_STEP: 77450 + | > loss_text_ce: 0.03309126943349838 (0.032450188430584066) + | > loss_mel_ce: 3.1794447898864746 (3.1068474422150696) + | > loss: 0.803134024143219 (0.7848244076291702) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2449 (0.24484549672878372) + | > loader_time: 0.0103 (0.011700573389187242) + + + --> TIME: 2024-10-20 08:11:16 -- STEP: 8441/69059 -- GLOBAL_STEP: 77500 + | > loss_text_ce: 0.0339013896882534 (0.03244828792683236) + | > loss_mel_ce: 2.970606565475464 (3.106886090436911) + | > loss: 0.7511270046234131 (0.7848335945652835) + | > current_lr: 5e-06 + | > step_time: 0.3226 (0.2448182763851109) + | > loader_time: 0.0117 (0.011697052355810254) + + + --> TIME: 2024-10-20 08:11:34 -- STEP: 8491/69059 -- GLOBAL_STEP: 77550 + | > loss_text_ce: 0.032344020903110504 (0.03244798930372621) + | > loss_mel_ce: 3.1089205741882324 (3.1066151658717334) + | > loss: 0.7853161692619324 (0.7847657887818545) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1983 (0.2447609148630896) + | > loader_time: 0.0107 (0.011694428974376702) + + + --> TIME: 2024-10-20 08:11:52 -- STEP: 8541/69059 -- GLOBAL_STEP: 77600 + | > loss_text_ce: 0.03315570205450058 (0.03244513283052867) + | > loss_mel_ce: 2.9714720249176025 (3.1063274443393802) + | > loss: 0.7511569261550903 (0.7846931442766663) + | > current_lr: 5e-06 + | > step_time: 0.196 (0.24472231343484518) + | > loader_time: 0.0105 (0.011692298043459512) + + + --> TIME: 2024-10-20 08:12:12 -- STEP: 8591/69059 -- GLOBAL_STEP: 77650 + | > loss_text_ce: 0.03249560296535492 (0.03244525300276644) + | > loss_mel_ce: 3.1256210803985596 (3.1062943022904133) + | > loss: 0.7895291447639465 (0.7846848888025888) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2264 (0.24484050183700834) + | > loader_time: 0.0105 (0.011693173300719019) + + + --> TIME: 2024-10-20 08:12:31 -- STEP: 8641/69059 -- GLOBAL_STEP: 77700 + | > loss_text_ce: 0.03290599212050438 (0.03244396705193778) + | > loss_mel_ce: 3.0910823345184326 (3.106338667739887) + | > loss: 0.780997097492218 (0.7846956586909503) + | > current_lr: 5e-06 + | > step_time: 0.223 (0.24482861170544148) + | > loader_time: 0.0109 (0.011692245060916633) + + + --> TIME: 2024-10-20 08:12:49 -- STEP: 8691/69059 -- GLOBAL_STEP: 77750 + | > loss_text_ce: 0.03435131534934044 (0.032445436688389805) + | > loss_mel_ce: 2.791147232055664 (3.1064426239745777) + | > loss: 0.7063746452331543 (0.7847220151639204) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1949 (0.2447894259819213) + | > loader_time: 0.0099 (0.011688247016131598) + + + --> TIME: 2024-10-20 08:13:08 -- STEP: 8741/69059 -- GLOBAL_STEP: 77800 + | > loss_text_ce: 0.03062446229159832 (0.03244418275054903) + | > loss_mel_ce: 2.71567702293396 (3.1062513406974115) + | > loss: 0.6865753531455994 (0.7846738808475004) + | > current_lr: 5e-06 + | > step_time: 0.2233 (0.24484311294642952) + | > loader_time: 0.011 (0.011685400120440918) + + + --> TIME: 2024-10-20 08:13:25 -- STEP: 8791/69059 -- GLOBAL_STEP: 77850 + | > loss_text_ce: 0.033183734863996506 (0.03244449015858917) + | > loss_mel_ce: 3.334636688232422 (3.106142097929203) + | > loss: 0.8419551253318787 (0.7846466469935571) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2471 (0.2447788138775245) + | > loader_time: 0.0118 (0.011681908485005573) + + + --> TIME: 2024-10-20 08:13:46 -- STEP: 8841/69059 -- GLOBAL_STEP: 77900 + | > loss_text_ce: 0.03445565700531006 (0.03244498922844487) + | > loss_mel_ce: 3.475247383117676 (3.1060224964451177) + | > loss: 0.8774257898330688 (0.7846168713957437) + | > current_lr: 5e-06 + | > step_time: 0.4195 (0.24493948235634216) + | > loader_time: 0.0125 (0.011681367228870124) + + + --> TIME: 2024-10-20 08:14:04 -- STEP: 8891/69059 -- GLOBAL_STEP: 77950 + | > loss_text_ce: 0.032808516174554825 (0.0324418112261031) + | > loss_mel_ce: 3.2082650661468506 (3.1059940835244584) + | > loss: 0.8102684020996094 (0.7846089736533883) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4422 (0.24489246939475812) + | > loader_time: 0.0135 (0.01167854258775739) + + + --> TIME: 2024-10-20 08:14:23 -- STEP: 8941/69059 -- GLOBAL_STEP: 78000 + | > loss_text_ce: 0.03165560960769653 (0.03244257497186336) + | > loss_mel_ce: 2.913173198699951 (3.10626455855148) + | > loss: 0.7362071871757507 (0.7846767833643267) + | > current_lr: 5e-06 + | > step_time: 0.1864 (0.24492563701105288) + | > loader_time: 0.0104 (0.011677205542592252) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_78000.pth + + --> TIME: 2024-10-20 08:14:55 -- STEP: 8991/69059 -- GLOBAL_STEP: 78050 + | > loss_text_ce: 0.030384168028831482 (0.032441456398523055) + | > loss_mel_ce: 2.76499342918396 (3.105974058356593) + | > loss: 0.6988443732261658 (0.7846038786684256) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.24486383600503348) + | > loader_time: 0.0108 (0.011675547418519073) + + + --> TIME: 2024-10-20 08:15:13 -- STEP: 9041/69059 -- GLOBAL_STEP: 78100 + | > loss_text_ce: 0.035135719925165176 (0.03244186484725352) + | > loss_mel_ce: 3.109700918197632 (3.1057223857813785) + | > loss: 0.7862091660499573 (0.7845410626555105) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.24480480673618785) + | > loader_time: 0.0104 (0.011673852440372367) + + + --> TIME: 2024-10-20 08:15:32 -- STEP: 9091/69059 -- GLOBAL_STEP: 78150 + | > loss_text_ce: 0.032596684992313385 (0.032443016263475115) + | > loss_mel_ce: 2.9389452934265137 (3.1059584584570237) + | > loss: 0.7428854703903198 (0.7846003686709051) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2143 (0.24480586672967428) + | > loader_time: 0.0104 (0.011673187871221133) + + + --> TIME: 2024-10-20 08:15:49 -- STEP: 9141/69059 -- GLOBAL_STEP: 78200 + | > loss_text_ce: 0.03060426004230976 (0.032441732871121315) + | > loss_mel_ce: 3.092822313308716 (3.1054832298663557) + | > loss: 0.7808566689491272 (0.7844812406477427) + | > current_lr: 5e-06 + | > step_time: 0.1958 (0.2447196145689393) + | > loader_time: 0.0108 (0.011671749302157051) + + + --> TIME: 2024-10-20 08:16:08 -- STEP: 9191/69059 -- GLOBAL_STEP: 78250 + | > loss_text_ce: 0.031723905354738235 (0.03243953039432529) + | > loss_mel_ce: 2.8610310554504395 (3.1054722886650463) + | > loss: 0.7231887578964233 (0.7844779547326214) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2065 (0.24472973051248603) + | > loader_time: 0.0109 (0.011671425299861869) + + + --> TIME: 2024-10-20 08:16:27 -- STEP: 9241/69059 -- GLOBAL_STEP: 78300 + | > loss_text_ce: 0.03337542340159416 (0.03243993508026121) + | > loss_mel_ce: 3.1601428985595703 (3.105558322500501) + | > loss: 0.7983796000480652 (0.7844995643369904) + | > current_lr: 5e-06 + | > step_time: 0.3711 (0.24481411597362632) + | > loader_time: 0.0137 (0.011672232318990061) + + + --> TIME: 2024-10-20 08:16:46 -- STEP: 9291/69059 -- GLOBAL_STEP: 78350 + | > loss_text_ce: 0.03169896453619003 (0.0324414438626094) + | > loss_mel_ce: 3.2487783432006836 (3.1055537321995805) + | > loss: 0.820119321346283 (0.7844987939792617) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24481911305628842) + | > loader_time: 0.0105 (0.01167267840024847) + + + --> TIME: 2024-10-20 08:17:05 -- STEP: 9341/69059 -- GLOBAL_STEP: 78400 + | > loss_text_ce: 0.03364318609237671 (0.032443080133928076) + | > loss_mel_ce: 3.433142900466919 (3.1055279606836605) + | > loss: 0.8666965365409851 (0.7844927601596814) + | > current_lr: 5e-06 + | > step_time: 0.2557 (0.24488239698754102) + | > loader_time: 0.0122 (0.011671675488188482) + + + --> TIME: 2024-10-20 08:17:23 -- STEP: 9391/69059 -- GLOBAL_STEP: 78450 + | > loss_text_ce: 0.03559333086013794 (0.03244309360820916) + | > loss_mel_ce: 2.9409725666046143 (3.105353597304447) + | > loss: 0.7441414594650269 (0.7844491726597366) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1975 (0.24483232484967196) + | > loader_time: 0.0105 (0.011669284783794857) + + + --> TIME: 2024-10-20 08:17:41 -- STEP: 9441/69059 -- GLOBAL_STEP: 78500 + | > loss_text_ce: 0.03324071690440178 (0.032441943899087444) + | > loss_mel_ce: 2.8174734115600586 (3.1050926705132023) + | > loss: 0.7126785516738892 (0.7843836535464006) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.2447684577127317) + | > loader_time: 0.0104 (0.011669213281144002) + + + --> TIME: 2024-10-20 08:17:59 -- STEP: 9491/69059 -- GLOBAL_STEP: 78550 + | > loss_text_ce: 0.032170429825782776 (0.032440616010625735) + | > loss_mel_ce: 3.3956966400146484 (3.1049914081962564) + | > loss: 0.8569667935371399 (0.7843580060080051) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4126 (0.24477689146380097) + | > loader_time: 0.0167 (0.011668944532157098) + + + --> TIME: 2024-10-20 08:18:17 -- STEP: 9541/69059 -- GLOBAL_STEP: 78600 + | > loss_text_ce: 0.03127516061067581 (0.03243777295501333) + | > loss_mel_ce: 3.041276454925537 (3.1050815772940306) + | > loss: 0.7681379318237305 (0.7843798375107212) + | > current_lr: 5e-06 + | > step_time: 0.1861 (0.2447231516574692) + | > loader_time: 0.0105 (0.011667640488323437) + + + --> TIME: 2024-10-20 08:18:37 -- STEP: 9591/69059 -- GLOBAL_STEP: 78650 + | > loss_text_ce: 0.03205348923802376 (0.03243745925176247) + | > loss_mel_ce: 2.8927559852600098 (3.1051306700186947) + | > loss: 0.7312023639678955 (0.7843920322506612) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2274 (0.24478454545595493) + | > loader_time: 0.0107 (0.011667103902128518) + + + --> TIME: 2024-10-20 08:18:55 -- STEP: 9641/69059 -- GLOBAL_STEP: 78700 + | > loss_text_ce: 0.033273372799158096 (0.03243820317253389) + | > loss_mel_ce: 2.6360247135162354 (3.1050703894015044) + | > loss: 0.6673245429992676 (0.7843771480806709) + | > current_lr: 5e-06 + | > step_time: 0.1706 (0.24477340081781415) + | > loader_time: 0.0103 (0.011664157313883134) + + + --> TIME: 2024-10-20 08:19:13 -- STEP: 9691/69059 -- GLOBAL_STEP: 78750 + | > loss_text_ce: 0.033275239169597626 (0.03243713831226684) + | > loss_mel_ce: 3.0707950592041016 (3.1047358415946307) + | > loss: 0.7760175466537476 (0.7842932449148824) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2551 (0.24473750420293752) + | > loader_time: 0.0105 (0.01166139959292621) + + + --> TIME: 2024-10-20 08:19:31 -- STEP: 9741/69059 -- GLOBAL_STEP: 78800 + | > loss_text_ce: 0.030834347009658813 (0.032436454006265276) + | > loss_mel_ce: 3.0155093669891357 (3.104850851159215) + | > loss: 0.7615859508514404 (0.7843218262393103) + | > current_lr: 5e-06 + | > step_time: 0.1955 (0.2447048456146786) + | > loader_time: 0.0106 (0.011660689825933851) + + + --> TIME: 2024-10-20 08:19:50 -- STEP: 9791/69059 -- GLOBAL_STEP: 78850 + | > loss_text_ce: 0.03239324316382408 (0.032435819403344585) + | > loss_mel_ce: 3.0832176208496094 (3.10480563361291) + | > loss: 0.7789027094841003 (0.7843103631927577) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3969 (0.24475637954000487) + | > loader_time: 0.0127 (0.011659599156536056) + + + --> TIME: 2024-10-20 08:20:10 -- STEP: 9841/69059 -- GLOBAL_STEP: 78900 + | > loss_text_ce: 0.03152577951550484 (0.032436187145403465) + | > loss_mel_ce: 3.1841330528259277 (3.105045785943665) + | > loss: 0.8039147257804871 (0.7843704931971713) + | > current_lr: 5e-06 + | > step_time: 0.2313 (0.24483753377230066) + | > loader_time: 0.0122 (0.01166005457287072) + + + --> TIME: 2024-10-20 08:20:28 -- STEP: 9891/69059 -- GLOBAL_STEP: 78950 + | > loss_text_ce: 0.029588136821985245 (0.032435136143927554) + | > loss_mel_ce: 3.37148118019104 (3.10491196791765) + | > loss: 0.8502673506736755 (0.7843367759360174) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2276 (0.24477034402850573) + | > loader_time: 0.0111 (0.011658026974189903) + + + --> TIME: 2024-10-20 08:20:45 -- STEP: 9941/69059 -- GLOBAL_STEP: 79000 + | > loss_text_ce: 0.03172309696674347 (0.03243348239494302) + | > loss_mel_ce: 3.384984254837036 (3.1045766376343424) + | > loss: 0.8541768193244934 (0.7842525299460502) + | > current_lr: 5e-06 + | > step_time: 0.2253 (0.24472769210185344) + | > loader_time: 0.0106 (0.01165683513411504) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_79000.pth + + --> TIME: 2024-10-20 08:21:25 -- STEP: 9991/69059 -- GLOBAL_STEP: 79050 + | > loss_text_ce: 0.03285632282495499 (0.03243203500656563) + | > loss_mel_ce: 3.192963123321533 (3.1044737889165126) + | > loss: 0.8064548373222351 (0.7842264559256775) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4698 (0.2447587031523944) + | > loader_time: 0.0131 (0.011656241330545915) + + + --> TIME: 2024-10-20 08:21:43 -- STEP: 10041/69059 -- GLOBAL_STEP: 79100 + | > loss_text_ce: 0.0334366038441658 (0.032430901235814914) + | > loss_mel_ce: 2.847423791885376 (3.1042279496918104) + | > loss: 0.7202150821685791 (0.7841647127070939) + | > current_lr: 5e-06 + | > step_time: 0.1945 (0.24474669837248827) + | > loader_time: 0.0108 (0.01165498567403951) + + + --> TIME: 2024-10-20 08:22:01 -- STEP: 10091/69059 -- GLOBAL_STEP: 79150 + | > loss_text_ce: 0.03301030024886131 (0.032431613298941786) + | > loss_mel_ce: 3.1644201278686523 (3.104106325139708) + | > loss: 0.7993575930595398 (0.784134484580174) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2309 (0.24472182324615233) + | > loader_time: 0.0108 (0.011653880559822813) + + + --> TIME: 2024-10-20 08:22:19 -- STEP: 10141/69059 -- GLOBAL_STEP: 79200 + | > loss_text_ce: 0.032219067215919495 (0.03243024831186099) + | > loss_mel_ce: 3.2784366607666016 (3.103972735695087) + | > loss: 0.8276639580726624 (0.7841007459588017) + | > current_lr: 5e-06 + | > step_time: 0.4922 (0.24462465564711613) + | > loader_time: 0.0142 (0.011653717165338447) + + + --> TIME: 2024-10-20 08:22:37 -- STEP: 10191/69059 -- GLOBAL_STEP: 79250 + | > loss_text_ce: 0.03155900165438652 (0.03242822894381493) + | > loss_mel_ce: 3.1670782566070557 (3.1037409184854243) + | > loss: 0.7996593117713928 (0.7840422868213935) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4978 (0.2446070321934367) + | > loader_time: 0.0135 (0.011654218084713229) + + + --> TIME: 2024-10-20 08:22:55 -- STEP: 10241/69059 -- GLOBAL_STEP: 79300 + | > loss_text_ce: 0.033157285302877426 (0.03242701714147821) + | > loss_mel_ce: 2.9053189754486084 (3.103777196442007) + | > loss: 0.7346190810203552 (0.7840510533664057) + | > current_lr: 5e-06 + | > step_time: 0.1762 (0.24462136008429608) + | > loader_time: 0.0102 (0.011653855843139414) + + + --> TIME: 2024-10-20 08:23:13 -- STEP: 10291/69059 -- GLOBAL_STEP: 79350 + | > loss_text_ce: 0.03401517495512962 (0.03242551664582088) + | > loss_mel_ce: 2.9186909198760986 (3.1035705154392237) + | > loss: 0.7381765246391296 (0.7839990079941564) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1953 (0.24453575550183512) + | > loader_time: 0.0104 (0.011652238166734228) + + + --> TIME: 2024-10-20 08:23:31 -- STEP: 10341/69059 -- GLOBAL_STEP: 79400 + | > loss_text_ce: 0.03486540541052818 (0.03242487919470677) + | > loss_mel_ce: 2.94197678565979 (3.1034311612815997) + | > loss: 0.7442105412483215 (0.7839640100785481) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.244554299218156) + | > loader_time: 0.0107 (0.011651956047129694) + + + --> TIME: 2024-10-20 08:23:52 -- STEP: 10391/69059 -- GLOBAL_STEP: 79450 + | > loss_text_ce: 0.031768690794706345 (0.03242433277800297) + | > loss_mel_ce: 3.026623487472534 (3.1033905230935077) + | > loss: 0.7645980715751648 (0.7839537139187157) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.24471018053021223) + | > loader_time: 0.0109 (0.011653288348486794) + + + --> TIME: 2024-10-20 08:24:11 -- STEP: 10441/69059 -- GLOBAL_STEP: 79500 + | > loss_text_ce: 0.03417176753282547 (0.0324255388903358) + | > loss_mel_ce: 3.180399179458618 (3.1033666323409794) + | > loss: 0.803642749786377 (0.7839480427567171) + | > current_lr: 5e-06 + | > step_time: 0.447 (0.24476988888051754) + | > loader_time: 0.0128 (0.01165213391717147) + + + --> TIME: 2024-10-20 08:24:29 -- STEP: 10491/69059 -- GLOBAL_STEP: 79550 + | > loss_text_ce: 0.03077578917145729 (0.03242363110958788) + | > loss_mel_ce: 2.788417339324951 (3.1034459937528975) + | > loss: 0.7047982811927795 (0.783967406168393) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2141 (0.24468390190070702) + | > loader_time: 0.0105 (0.011649445848371355) + + + --> TIME: 2024-10-20 08:24:48 -- STEP: 10541/69059 -- GLOBAL_STEP: 79600 + | > loss_text_ce: 0.03119574673473835 (0.0324233537217355) + | > loss_mel_ce: 2.865530252456665 (3.1034986981177575) + | > loss: 0.7241814732551575 (0.7839805129107925) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24473761030714108) + | > loader_time: 0.0105 (0.011648368681645394) + + + --> TIME: 2024-10-20 08:25:08 -- STEP: 10591/69059 -- GLOBAL_STEP: 79650 + | > loss_text_ce: 0.03478732332587242 (0.03242515000509336) + | > loss_mel_ce: 3.0630900859832764 (3.1035517621069597) + | > loss: 0.7744693756103516 (0.78399422797701) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4883 (0.24488195014646022) + | > loader_time: 0.0142 (0.011648524461482766) + + + --> TIME: 2024-10-20 08:25:26 -- STEP: 10641/69059 -- GLOBAL_STEP: 79700 + | > loss_text_ce: 0.03128218650817871 (0.03242374808292896) + | > loss_mel_ce: 2.7586817741394043 (3.1035143454130187) + | > loss: 0.6974909901618958 (0.7839845233243175) + | > current_lr: 5e-06 + | > step_time: 0.1779 (0.24483091606478555) + | > loader_time: 0.0104 (0.011646786462923466) + + + --> TIME: 2024-10-20 08:25:45 -- STEP: 10691/69059 -- GLOBAL_STEP: 79750 + | > loss_text_ce: 0.03214798495173454 (0.032423847320016456) + | > loss_mel_ce: 3.297307014465332 (3.1034529796318804) + | > loss: 0.8323637247085571 (0.7839692067067435) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2175 (0.24484892824101792) + | > loader_time: 0.011 (0.011646011861177282) + + + --> TIME: 2024-10-20 08:26:03 -- STEP: 10741/69059 -- GLOBAL_STEP: 79800 + | > loss_text_ce: 0.029607398435473442 (0.03242164471185586) + | > loss_mel_ce: 3.0522568225860596 (3.1032843036188122) + | > loss: 0.7704660296440125 (0.7839264870522321) + | > current_lr: 5e-06 + | > step_time: 0.1723 (0.24481735053247566) + | > loader_time: 0.011 (0.011644830717941495) + + + --> TIME: 2024-10-20 08:26:22 -- STEP: 10791/69059 -- GLOBAL_STEP: 79850 + | > loss_text_ce: 0.03024645522236824 (0.032421076587608545) + | > loss_mel_ce: 3.342928171157837 (3.1033068596957696) + | > loss: 0.8432936668395996 (0.7839319840461605) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3568 (0.24484839332014252) + | > loader_time: 0.0123 (0.011644709995132991) + + + --> TIME: 2024-10-20 08:26:41 -- STEP: 10841/69059 -- GLOBAL_STEP: 79900 + | > loss_text_ce: 0.030632296577095985 (0.03241934385001205) + | > loss_mel_ce: 3.441359043121338 (3.10324534546481) + | > loss: 0.8679978251457214 (0.7839161722975204) + | > current_lr: 5e-06 + | > step_time: 0.2559 (0.2449056799369329) + | > loader_time: 0.0107 (0.011643739327679802) + + + --> TIME: 2024-10-20 08:26:59 -- STEP: 10891/69059 -- GLOBAL_STEP: 79950 + | > loss_text_ce: 0.032717373222112656 (0.032418052841447685) + | > loss_mel_ce: 3.1711058616638184 (3.1030885977101876) + | > loss: 0.8009558320045471 (0.7838766625975887) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2549 (0.24486947107792298) + | > loader_time: 0.0109 (0.011642110083957189) + + + --> TIME: 2024-10-20 08:27:15 -- STEP: 10941/69059 -- GLOBAL_STEP: 80000 + | > loss_text_ce: 0.028356801718473434 (0.03241441592903728) + | > loss_mel_ce: 2.9273386001586914 (3.103087452950462) + | > loss: 0.7389238476753235 (0.783875467174504) + | > current_lr: 5e-06 + | > step_time: 0.1721 (0.24471982514092286) + | > loader_time: 0.0102 (0.011638021787269179) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_80000.pth + + --> TIME: 2024-10-20 08:27:48 -- STEP: 10991/69059 -- GLOBAL_STEP: 80050 + | > loss_text_ce: 0.034101102501153946 (0.032414670437255905) + | > loss_mel_ce: 3.126004219055176 (3.1030504787904234) + | > loss: 0.7900263071060181 (0.7838662872756096) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2882 (0.2447098196284937) + | > loader_time: 0.012 (0.011649308500965886) + + + --> TIME: 2024-10-20 08:28:06 -- STEP: 11041/69059 -- GLOBAL_STEP: 80100 + | > loss_text_ce: 0.03203582391142845 (0.032412444424471686) + | > loss_mel_ce: 3.1115243434906006 (3.1028961992363113) + | > loss: 0.7858900427818298 (0.7838271608949507) + | > current_lr: 5e-06 + | > step_time: 0.2059 (0.244656247080328) + | > loader_time: 0.0109 (0.011647577638179578) + + + --> TIME: 2024-10-20 08:28:23 -- STEP: 11091/69059 -- GLOBAL_STEP: 80150 + | > loss_text_ce: 0.03472267463803291 (0.03241180631762634) + | > loss_mel_ce: 3.060317277908325 (3.1027870777547317) + | > loss: 0.7737599611282349 (0.7837997209970962) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2285 (0.24457289503818275) + | > loader_time: 0.0108 (0.011645887252949363) + + + --> TIME: 2024-10-20 08:28:42 -- STEP: 11141/69059 -- GLOBAL_STEP: 80200 + | > loss_text_ce: 0.03142033517360687 (0.032411517600795955) + | > loss_mel_ce: 3.0068359375 (3.102811054335537) + | > loss: 0.7595640420913696 (0.7838056429681579) + | > current_lr: 5e-06 + | > step_time: 0.2344 (0.24459354454391835) + | > loader_time: 0.0112 (0.01164557024088878) + + + --> TIME: 2024-10-20 08:29:02 -- STEP: 11191/69059 -- GLOBAL_STEP: 80250 + | > loss_text_ce: 0.03227972239255905 (0.03241186657925989) + | > loss_mel_ce: 3.049067497253418 (3.102955399308635) + | > loss: 0.7703368067741394 (0.7838418164379773) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2146 (0.24466537732603344) + | > loader_time: 0.0109 (0.011646942589134112) + + + --> TIME: 2024-10-20 08:29:21 -- STEP: 11241/69059 -- GLOBAL_STEP: 80300 + | > loss_text_ce: 0.03470172733068466 (0.032412086623998436) + | > loss_mel_ce: 3.366443395614624 (3.1029659972773262) + | > loss: 0.8502863049507141 (0.7838445209373022) + | > current_lr: 5e-06 + | > step_time: 0.313 (0.24469854063691224) + | > loader_time: 0.0131 (0.011648269853879564) + + + --> TIME: 2024-10-20 08:29:39 -- STEP: 11291/69059 -- GLOBAL_STEP: 80350 + | > loss_text_ce: 0.032032109797000885 (0.03241070441769062) + | > loss_mel_ce: 2.704400062561035 (3.1030813225947247) + | > loss: 0.6841080188751221 (0.7838730067107066) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2823 (0.24468296503917228) + | > loader_time: 0.012 (0.011647996694686318) + + + --> TIME: 2024-10-20 08:29:58 -- STEP: 11341/69059 -- GLOBAL_STEP: 80400 + | > loss_text_ce: 0.032066378742456436 (0.032408718799699504) + | > loss_mel_ce: 3.452255964279175 (3.1027888555226517) + | > loss: 0.8710805773735046 (0.7837993935527073) + | > current_lr: 5e-06 + | > step_time: 0.1951 (0.24469391158749512) + | > loader_time: 0.0104 (0.011648115789267006) + + + --> TIME: 2024-10-20 08:30:17 -- STEP: 11391/69059 -- GLOBAL_STEP: 80450 + | > loss_text_ce: 0.03143378347158432 (0.032408024553059536) + | > loss_mel_ce: 3.2870686054229736 (3.1028442952748216) + | > loss: 0.8296256065368652 (0.783813079941231) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.383 (0.24474055904572284) + | > loader_time: 0.0126 (0.011646857683196706) + + + --> TIME: 2024-10-20 08:30:34 -- STEP: 11441/69059 -- GLOBAL_STEP: 80500 + | > loss_text_ce: 0.03006620891392231 (0.03240507739992426) + | > loss_mel_ce: 3.033062696456909 (3.102742806419543) + | > loss: 0.7657822370529175 (0.7837869709224273) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.24465285238608253) + | > loader_time: 0.0108 (0.011645711496695115) + + + --> TIME: 2024-10-20 08:30:53 -- STEP: 11491/69059 -- GLOBAL_STEP: 80550 + | > loss_text_ce: 0.03220275044441223 (0.032404592820598016) + | > loss_mel_ce: 3.565258741378784 (3.1027039554533857) + | > loss: 0.8993653655052185 (0.7837771370354681) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4692 (0.24469747898960623) + | > loader_time: 0.0134 (0.011644484075364217) + + + --> TIME: 2024-10-20 08:31:12 -- STEP: 11541/69059 -- GLOBAL_STEP: 80600 + | > loss_text_ce: 0.03447418659925461 (0.03240325228709694) + | > loss_mel_ce: 3.244992733001709 (3.1024760278999746) + | > loss: 0.8198667168617249 (0.7837198200232844) + | > current_lr: 5e-06 + | > step_time: 0.3042 (0.24469788625452904) + | > loader_time: 0.012 (0.011643599621004561) + + + --> TIME: 2024-10-20 08:31:32 -- STEP: 11591/69059 -- GLOBAL_STEP: 80650 + | > loss_text_ce: 0.0337681919336319 (0.03240336076960784) + | > loss_mel_ce: 3.339327096939087 (3.1024534282687606) + | > loss: 0.8432738184928894 (0.7837141972334379) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4169 (0.24481501810483433) + | > loader_time: 0.0127 (0.011644258479399034) + + + --> TIME: 2024-10-20 08:31:49 -- STEP: 11641/69059 -- GLOBAL_STEP: 80700 + | > loss_text_ce: 0.03485625982284546 (0.03240142553083411) + | > loss_mel_ce: 2.933136463165283 (3.1021563410195734) + | > loss: 0.7419981956481934 (0.7836394415883989) + | > current_lr: 5e-06 + | > step_time: 0.5317 (0.24471046694999773) + | > loader_time: 0.0155 (0.011641279199608615) + + + --> TIME: 2024-10-20 08:32:07 -- STEP: 11691/69059 -- GLOBAL_STEP: 80750 + | > loss_text_ce: 0.0310807004570961 (0.032400586056414715) + | > loss_mel_ce: 3.2179036140441895 (3.102243018959944) + | > loss: 0.8122460842132568 (0.783660901208682) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.244723581735495) + | > loader_time: 0.0102 (0.011639531934733468) + + + --> TIME: 2024-10-20 08:32:23 -- STEP: 11741/69059 -- GLOBAL_STEP: 80800 + | > loss_text_ce: 0.03322482109069824 (0.03239778687384415) + | > loss_mel_ce: 3.018345355987549 (3.10226943320081) + | > loss: 0.7628925442695618 (0.7836668049798743) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24456427915875448) + | > loader_time: 0.0107 (0.01163561365874684) + + + --> TIME: 2024-10-20 08:32:42 -- STEP: 11791/69059 -- GLOBAL_STEP: 80850 + | > loss_text_ce: 0.03376776725053787 (0.03239757386782252) + | > loss_mel_ce: 2.899440050125122 (3.10213837766352) + | > loss: 0.7333019375801086 (0.7836339878334293) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3605 (0.24455397873775683) + | > loader_time: 0.0107 (0.011634243726629283) + + + --> TIME: 2024-10-20 08:33:01 -- STEP: 11841/69059 -- GLOBAL_STEP: 80900 + | > loss_text_ce: 0.03298189491033554 (0.03239741655752945) + | > loss_mel_ce: 3.399955987930298 (3.1022707503265377) + | > loss: 0.8582344651222229 (0.7836670416823585) + | > current_lr: 5e-06 + | > step_time: 0.304 (0.24459895625749503) + | > loader_time: 0.0116 (0.011634357736054549) + + + --> TIME: 2024-10-20 08:33:19 -- STEP: 11891/69059 -- GLOBAL_STEP: 80950 + | > loss_text_ce: 0.030230939388275146 (0.03239672566989558) + | > loss_mel_ce: 3.030050039291382 (3.102355876388786) + | > loss: 0.7650702595710754 (0.7836881504709663) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2042 (0.2445530575224614) + | > loader_time: 0.0107 (0.011633617464321754) + + + --> TIME: 2024-10-20 08:33:38 -- STEP: 11941/69059 -- GLOBAL_STEP: 81000 + | > loss_text_ce: 0.03146102651953697 (0.032398476705645315) + | > loss_mel_ce: 3.2032175064086914 (3.102380592073689) + | > loss: 0.8086696267127991 (0.7836947671463209) + | > current_lr: 5e-06 + | > step_time: 0.2834 (0.2446245139652077) + | > loader_time: 0.0115 (0.011632201959315958) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_81000.pth + + --> TIME: 2024-10-20 08:34:11 -- STEP: 11991/69059 -- GLOBAL_STEP: 81050 + | > loss_text_ce: 0.03420979529619217 (0.032395106402205186) + | > loss_mel_ce: 2.8667328357696533 (3.102272442806632) + | > loss: 0.725235641002655 (0.7836668872473358) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2227 (0.24456710241204976) + | > loader_time: 0.0107 (0.011631552545354895) + + + --> TIME: 2024-10-20 08:34:29 -- STEP: 12041/69059 -- GLOBAL_STEP: 81100 + | > loss_text_ce: 0.03070632554590702 (0.032394478989511644) + | > loss_mel_ce: 2.924835443496704 (3.1021666295919856) + | > loss: 0.7388854622840881 (0.7836402770976125) + | > current_lr: 5e-06 + | > step_time: 0.187 (0.24456840529313925) + | > loader_time: 0.0115 (0.011631829369732493) + + + --> TIME: 2024-10-20 08:34:47 -- STEP: 12091/69059 -- GLOBAL_STEP: 81150 + | > loss_text_ce: 0.031748153269290924 (0.03239495336461634) + | > loss_mel_ce: 2.8155860900878906 (3.1018779499833857) + | > loss: 0.7118335366249084 (0.7835682257806552) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2048 (0.2445397204081323) + | > loader_time: 0.0108 (0.011630834258132177) + + + --> TIME: 2024-10-20 08:35:06 -- STEP: 12141/69059 -- GLOBAL_STEP: 81200 + | > loss_text_ce: 0.033587921410799026 (0.03239445283072529) + | > loss_mel_ce: 3.083083391189575 (3.1018442217830993) + | > loss: 0.7791678309440613 (0.7835596685904008) + | > current_lr: 5e-06 + | > step_time: 0.2061 (0.2445298149444138) + | > loader_time: 0.0109 (0.011630976772536182) + + + --> TIME: 2024-10-20 08:35:24 -- STEP: 12191/69059 -- GLOBAL_STEP: 81250 + | > loss_text_ce: 0.035667534917593 (0.03239508175058359) + | > loss_mel_ce: 2.8589890003204346 (3.10177283746245) + | > loss: 0.7236641049385071 (0.7835419797328224) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3691 (0.24454675347759458) + | > loader_time: 0.0123 (0.011629998503027844) + + + --> TIME: 2024-10-20 08:35:43 -- STEP: 12241/69059 -- GLOBAL_STEP: 81300 + | > loss_text_ce: 0.03297463059425354 (0.03239405327553797) + | > loss_mel_ce: 3.445324182510376 (3.1018434992102204) + | > loss: 0.8695747256278992 (0.7835593880484) + | > current_lr: 5e-06 + | > step_time: 0.2157 (0.24454620404333458) + | > loader_time: 0.0107 (0.01162968859467262) + + + --> TIME: 2024-10-20 08:36:02 -- STEP: 12291/69059 -- GLOBAL_STEP: 81350 + | > loss_text_ce: 0.03264034539461136 (0.03239449999072337) + | > loss_mel_ce: 3.691987991333008 (3.101871334054511) + | > loss: 0.931157112121582 (0.7835664584458025) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3146 (0.24458669733549193) + | > loader_time: 0.0127 (0.011628796363534515) + + + --> TIME: 2024-10-20 08:36:20 -- STEP: 12341/69059 -- GLOBAL_STEP: 81400 + | > loss_text_ce: 0.031651757657527924 (0.0323949715676651) + | > loss_mel_ce: 3.292937755584717 (3.101825230048778) + | > loss: 0.8311473727226257 (0.783555050340002) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.2445824853740841) + | > loader_time: 0.0103 (0.011627388603248197) + + + --> TIME: 2024-10-20 08:36:39 -- STEP: 12391/69059 -- GLOBAL_STEP: 81450 + | > loss_text_ce: 0.03627382963895798 (0.03239476248247952) + | > loss_mel_ce: 3.220712423324585 (3.1016418355860638) + | > loss: 0.8142465353012085 (0.7835091494655747) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4107 (0.2446256272296969) + | > loader_time: 0.0127 (0.01162773074908272) + + + --> TIME: 2024-10-20 08:36:58 -- STEP: 12441/69059 -- GLOBAL_STEP: 81500 + | > loss_text_ce: 0.03456619009375572 (0.032392536624385876) + | > loss_mel_ce: 3.1563823223114014 (3.101521877832) + | > loss: 0.7977371215820312 (0.7834786035634911) + | > current_lr: 5e-06 + | > step_time: 0.4212 (0.244637056807883) + | > loader_time: 0.0151 (0.011626982150397128) + + + --> TIME: 2024-10-20 08:37:17 -- STEP: 12491/69059 -- GLOBAL_STEP: 81550 + | > loss_text_ce: 0.032916005700826645 (0.03239162625213645) + | > loss_mel_ce: 3.1674253940582275 (3.10141352786045) + | > loss: 0.8000853657722473 (0.7834512884732703) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2911 (0.24466380211400526) + | > loader_time: 0.011 (0.011625526770418537) + + + --> TIME: 2024-10-20 08:37:35 -- STEP: 12541/69059 -- GLOBAL_STEP: 81600 + | > loss_text_ce: 0.03208418935537338 (0.032390232917467225) + | > loss_mel_ce: 2.600799798965454 (3.1013475727325845) + | > loss: 0.6582210063934326 (0.7834344513630537) + | > current_lr: 5e-06 + | > step_time: 0.1848 (0.24465451920377557) + | > loader_time: 0.0108 (0.011623671024232205) + + + --> TIME: 2024-10-20 08:37:54 -- STEP: 12591/69059 -- GLOBAL_STEP: 81650 + | > loss_text_ce: 0.029230128973722458 (0.032388332782377176) + | > loss_mel_ce: 2.827061176300049 (3.1012879156269144) + | > loss: 0.7140728235244751 (0.78341906206667) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2813 (0.24466840217835137) + | > loader_time: 0.0113 (0.011621536306084805) + + + --> TIME: 2024-10-20 08:38:12 -- STEP: 12641/69059 -- GLOBAL_STEP: 81700 + | > loss_text_ce: 0.031017722561955452 (0.0323862134303716) + | > loss_mel_ce: 3.3343355655670166 (3.101113505759916) + | > loss: 0.8413383364677429 (0.7833749297468462) + | > current_lr: 5e-06 + | > step_time: 0.2254 (0.2446210235934895) + | > loader_time: 0.0115 (0.01161960776155299) + + + --> TIME: 2024-10-20 08:38:29 -- STEP: 12691/69059 -- GLOBAL_STEP: 81750 + | > loss_text_ce: 0.031079692766070366 (0.032384381335621944) + | > loss_mel_ce: 3.1993494033813477 (3.101141470993356) + | > loss: 0.8076072931289673 (0.7833814630458819) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2007 (0.2445462939962974) + | > loader_time: 0.0103 (0.011617945606531406) + + + --> TIME: 2024-10-20 08:38:46 -- STEP: 12741/69059 -- GLOBAL_STEP: 81800 + | > loss_text_ce: 0.0341012142598629 (0.032383260914857145) + | > loss_mel_ce: 2.8040125370025635 (3.1009035914932825) + | > loss: 0.7095284461975098 (0.7833217130698352) + | > current_lr: 5e-06 + | > step_time: 0.1934 (0.2444790289695985) + | > loader_time: 0.0103 (0.011615751564535499) + + + --> TIME: 2024-10-20 08:39:04 -- STEP: 12791/69059 -- GLOBAL_STEP: 81850 + | > loss_text_ce: 0.031478725373744965 (0.032381681342398376) + | > loss_mel_ce: 3.036062717437744 (3.1008826478839273) + | > loss: 0.7668853402137756 (0.7833160822604551) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1936 (0.2444434364465877) + | > loader_time: 0.0112 (0.011614320444246217) + + + --> TIME: 2024-10-20 08:39:24 -- STEP: 12841/69059 -- GLOBAL_STEP: 81900 + | > loss_text_ce: 0.032205503433942795 (0.03238165075493655) + | > loss_mel_ce: 2.850933074951172 (3.100957624415223) + | > loss: 0.7207846641540527 (0.7833348187541723) + | > current_lr: 5e-06 + | > step_time: 0.1843 (0.24449795109954045) + | > loader_time: 0.0103 (0.011614830227862227) + + + --> TIME: 2024-10-20 08:39:42 -- STEP: 12891/69059 -- GLOBAL_STEP: 81950 + | > loss_text_ce: 0.03142934292554855 (0.03238127261591551) + | > loss_mel_ce: 2.9174537658691406 (3.100954172859253) + | > loss: 0.7372207641601562 (0.7833338613312597) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2137 (0.24447643180748335) + | > loader_time: 0.0102 (0.01161359486703075) + + + --> TIME: 2024-10-20 08:40:01 -- STEP: 12941/69059 -- GLOBAL_STEP: 82000 + | > loss_text_ce: 0.030557824298739433 (0.03238140219432053) + | > loss_mel_ce: 2.7719500064849854 (3.100987219108549) + | > loss: 0.7006269693374634 (0.7833421552819609) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24453783798011433) + | > loader_time: 0.0101 (0.011611804850880267) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_82000.pth + + --> TIME: 2024-10-20 08:40:40 -- STEP: 12991/69059 -- GLOBAL_STEP: 82050 + | > loss_text_ce: 0.032512515783309937 (0.03238046346684677) + | > loss_mel_ce: 3.19386887550354 (3.101068455520874) + | > loss: 0.8065953254699707 (0.7833622297002955) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2279 (0.24459277268563087) + | > loader_time: 0.0116 (0.01161118436837877) + + + --> TIME: 2024-10-20 08:40:59 -- STEP: 13041/69059 -- GLOBAL_STEP: 82100 + | > loss_text_ce: 0.030786829069256783 (0.03238054505978523) + | > loss_mel_ce: 2.8374602794647217 (3.100938445795405) + | > loss: 0.7170617580413818 (0.7833297476703767) + | > current_lr: 5e-06 + | > step_time: 0.2018 (0.24460387769245262) + | > loader_time: 0.0115 (0.011611870082735036) + + + --> TIME: 2024-10-20 08:41:18 -- STEP: 13091/69059 -- GLOBAL_STEP: 82150 + | > loss_text_ce: 0.03374999761581421 (0.032379058053310726) + | > loss_mel_ce: 3.1200435161590576 (3.10084906768857) + | > loss: 0.7884483933448792 (0.783307031380939) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2266 (0.24463619447156953) + | > loader_time: 0.0113 (0.011612066746523013) + + + --> TIME: 2024-10-20 08:41:37 -- STEP: 13141/69059 -- GLOBAL_STEP: 82200 + | > loss_text_ce: 0.0311304721981287 (0.03237890163203388) + | > loss_mel_ce: 3.2700836658477783 (3.1009073279036574) + | > loss: 0.8253035545349121 (0.7833215573291031) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.24465018209189318) + | > loader_time: 0.0102 (0.011611864761115698) + + + --> TIME: 2024-10-20 08:41:56 -- STEP: 13191/69059 -- GLOBAL_STEP: 82250 + | > loss_text_ce: 0.03154886141419411 (0.03237697200897827) + | > loss_mel_ce: 3.165304660797119 (3.100784384538275) + | > loss: 0.7992134094238281 (0.7832903390756353) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2374 (0.24468848127093712) + | > loader_time: 0.011 (0.01161194556170263) + + + --> TIME: 2024-10-20 08:42:15 -- STEP: 13241/69059 -- GLOBAL_STEP: 82300 + | > loss_text_ce: 0.035488229244947433 (0.032377044364813604) + | > loss_mel_ce: 3.0408401489257812 (3.100857183513895) + | > loss: 0.7690820693969727 (0.7833085569106645) + | > current_lr: 5e-06 + | > step_time: 0.2776 (0.24473717574419265) + | > loader_time: 0.0105 (0.01161209251862463) + + + --> TIME: 2024-10-20 08:42:33 -- STEP: 13291/69059 -- GLOBAL_STEP: 82350 + | > loss_text_ce: 0.032461512833833694 (0.03237541643063926) + | > loss_mel_ce: 3.159980535507202 (3.1007971939406636) + | > loss: 0.7981104850769043 (0.7832931525353313) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2947 (0.244717158238709) + | > loader_time: 0.0116 (0.011611753442506147) + + + --> TIME: 2024-10-20 08:42:52 -- STEP: 13341/69059 -- GLOBAL_STEP: 82400 + | > loss_text_ce: 0.03245460242033005 (0.03237511595624156) + | > loss_mel_ce: 2.949279546737671 (3.100563297135192) + | > loss: 0.745433509349823 (0.7832346032043053) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.24471775815812344) + | > loader_time: 0.0105 (0.011611190731133541) + + + --> TIME: 2024-10-20 08:43:11 -- STEP: 13391/69059 -- GLOBAL_STEP: 82450 + | > loss_text_ce: 0.0322323739528656 (0.032374081658575936) + | > loss_mel_ce: 2.974773406982422 (3.100394871706531) + | > loss: 0.7517514228820801 (0.7831922382968695) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2823 (0.24477386939303944) + | > loader_time: 0.0122 (0.011611616840267151) + + + --> TIME: 2024-10-20 08:43:29 -- STEP: 13441/69059 -- GLOBAL_STEP: 82500 + | > loss_text_ce: 0.031370148062705994 (0.03237386079851973) + | > loss_mel_ce: 3.1446917057037354 (3.1002126329667625) + | > loss: 0.7940154671669006 (0.7831466234064672) + | > current_lr: 5e-06 + | > step_time: 0.2066 (0.24471182702916447) + | > loader_time: 0.0109 (0.011611236542181743) + + + --> TIME: 2024-10-20 08:43:46 -- STEP: 13491/69059 -- GLOBAL_STEP: 82550 + | > loss_text_ce: 0.02935277484357357 (0.03237275451175449) + | > loss_mel_ce: 3.2724950313568115 (3.0999987632064916) + | > loss: 0.8254619240760803 (0.7830928793724707) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2596 (0.2446432882397605) + | > loader_time: 0.012 (0.011610283330994162) + + + --> TIME: 2024-10-20 08:44:04 -- STEP: 13541/69059 -- GLOBAL_STEP: 82600 + | > loss_text_ce: 0.03347277268767357 (0.032371211657242334) + | > loss_mel_ce: 3.1996326446533203 (3.099826267855301) + | > loss: 0.808276355266571 (0.7830493698205679) + | > current_lr: 5e-06 + | > step_time: 0.3864 (0.24461040415128943) + | > loader_time: 0.0125 (0.011609115326618532) + + + --> TIME: 2024-10-20 08:44:22 -- STEP: 13591/69059 -- GLOBAL_STEP: 82650 + | > loss_text_ce: 0.03214500471949577 (0.03236993432969266) + | > loss_mel_ce: 2.9087488651275635 (3.0998183293862187) + | > loss: 0.7352234721183777 (0.7830470658657287) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1905 (0.2445914600430696) + | > loader_time: 0.0104 (0.011608921782472784) + + + --> TIME: 2024-10-20 08:44:41 -- STEP: 13641/69059 -- GLOBAL_STEP: 82700 + | > loss_text_ce: 0.033556900918483734 (0.03236799755830578) + | > loss_mel_ce: 3.168856620788574 (3.099805385711106) + | > loss: 0.8006033897399902 (0.7830433457462794) + | > current_lr: 5e-06 + | > step_time: 0.4287 (0.2446276865592058) + | > loader_time: 0.0127 (0.011608279876088196) + + + --> TIME: 2024-10-20 08:44:59 -- STEP: 13691/69059 -- GLOBAL_STEP: 82750 + | > loss_text_ce: 0.02956029586493969 (0.032367542930522804) + | > loss_mel_ce: 3.0655417442321777 (3.0997002186606974) + | > loss: 0.7737755179405212 (0.7830169403120257) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2017 (0.24458934238381738) + | > loader_time: 0.0117 (0.011607480618579505) + + + --> TIME: 2024-10-20 08:45:17 -- STEP: 13741/69059 -- GLOBAL_STEP: 82800 + | > loss_text_ce: 0.03723890706896782 (0.03236709728896136) + | > loss_mel_ce: 3.316838502883911 (3.099600292728696) + | > loss: 0.8385193347930908 (0.7829918474160665) + | > current_lr: 5e-06 + | > step_time: 0.3482 (0.2445411364954638) + | > loader_time: 0.0125 (0.01160615209356408) + + + --> TIME: 2024-10-20 08:45:35 -- STEP: 13791/69059 -- GLOBAL_STEP: 82850 + | > loss_text_ce: 0.03182120993733406 (0.032366231770662286) + | > loss_mel_ce: 2.903249502182007 (3.099450277163632) + | > loss: 0.7337676882743835 (0.7829541271691145) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1955 (0.24451048461826105) + | > loader_time: 0.0104 (0.01160451515479344) + + + --> TIME: 2024-10-20 08:45:53 -- STEP: 13841/69059 -- GLOBAL_STEP: 82900 + | > loss_text_ce: 0.03170027211308479 (0.03236413436460921) + | > loss_mel_ce: 2.9032063484191895 (3.099585362816035) + | > loss: 0.7337266802787781 (0.7829873742511883) + | > current_lr: 5e-06 + | > step_time: 0.1782 (0.24448099271400708) + | > loader_time: 0.0111 (0.011603862200072057) + + + --> TIME: 2024-10-20 08:46:11 -- STEP: 13891/69059 -- GLOBAL_STEP: 82950 + | > loss_text_ce: 0.03175437077879906 (0.032361701226607524) + | > loss_mel_ce: 3.3347768783569336 (3.0996299793701185) + | > loss: 0.8416327834129333 (0.7829979201062051) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3113 (0.24450308618702754) + | > loader_time: 0.0119 (0.011604666649827584) + + + --> TIME: 2024-10-20 08:46:30 -- STEP: 13941/69059 -- GLOBAL_STEP: 83000 + | > loss_text_ce: 0.032288141548633575 (0.03236327732681872) + | > loss_mel_ce: 3.010568141937256 (3.0996879597372384) + | > loss: 0.760714054107666 (0.7830128092290708) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.24449473904636432) + | > loader_time: 0.0104 (0.011603924048253942) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_83000.pth + + --> TIME: 2024-10-20 08:47:06 -- STEP: 13991/69059 -- GLOBAL_STEP: 83050 + | > loss_text_ce: 0.03178082033991814 (0.032363067569665185) + | > loss_mel_ce: 3.173987627029419 (3.0996266067108977) + | > loss: 0.8014420866966248 (0.7829974185286364) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2046 (0.24455600454215798) + | > loader_time: 0.0109 (0.01160727142989447) + + + --> TIME: 2024-10-20 08:47:24 -- STEP: 14041/69059 -- GLOBAL_STEP: 83100 + | > loss_text_ce: 0.03307148814201355 (0.03236179389048953) + | > loss_mel_ce: 3.045393705368042 (3.0996238711926076) + | > loss: 0.7696163058280945 (0.7829964162235145) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.24451465727588273) + | > loader_time: 0.0113 (0.011607752932365238) + + + --> TIME: 2024-10-20 08:47:42 -- STEP: 14091/69059 -- GLOBAL_STEP: 83150 + | > loss_text_ce: 0.02808457985520363 (0.03236037161822275) + | > loss_mel_ce: 2.87445330619812 (3.0995176111727143) + | > loss: 0.7256344556808472 (0.7829694956505099) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1954 (0.24450109250482394) + | > loader_time: 0.0102 (0.011608618839207909) + + + --> TIME: 2024-10-20 08:48:01 -- STEP: 14141/69059 -- GLOBAL_STEP: 83200 + | > loss_text_ce: 0.03229561448097229 (0.03236041958714199) + | > loss_mel_ce: 3.320716142654419 (3.0995079776234364) + | > loss: 0.8382529616355896 (0.7829670992769917) + | > current_lr: 5e-06 + | > step_time: 0.2256 (0.24450351500492626) + | > loader_time: 0.0115 (0.01160819263591787) + + + --> TIME: 2024-10-20 08:48:19 -- STEP: 14191/69059 -- GLOBAL_STEP: 83250 + | > loss_text_ce: 0.03309417888522148 (0.03235846312998694) + | > loss_mel_ce: 3.0534896850585938 (3.0995265560366527) + | > loss: 0.7716459631919861 (0.7829712547582877) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2054 (0.2444844987493062) + | > loader_time: 0.0106 (0.011609444767079235) + + + --> TIME: 2024-10-20 08:48:40 -- STEP: 14241/69059 -- GLOBAL_STEP: 83300 + | > loss_text_ce: 0.02984522096812725 (0.032358574571967) + | > loss_mel_ce: 2.956606388092041 (3.099591130345115) + | > loss: 0.7466129064559937 (0.7829874261997104) + | > current_lr: 5e-06 + | > step_time: 0.178 (0.24460523308147988) + | > loader_time: 0.0103 (0.011611153642311446) + + + --> TIME: 2024-10-20 08:48:57 -- STEP: 14291/69059 -- GLOBAL_STEP: 83350 + | > loss_text_ce: 0.03227176517248154 (0.032356702747829494) + | > loss_mel_ce: 3.6179842948913574 (3.099493337673533) + | > loss: 0.9125640392303467 (0.7829625100829547) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3064 (0.24455660367627635) + | > loader_time: 0.0118 (0.011609953267772371) + + + --> TIME: 2024-10-20 08:49:15 -- STEP: 14341/69059 -- GLOBAL_STEP: 83400 + | > loss_text_ce: 0.034695714712142944 (0.032354723192223295) + | > loss_mel_ce: 3.036558151245117 (3.0994851067179066) + | > loss: 0.7678134441375732 (0.7829599574506462) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.2445298658821442) + | > loader_time: 0.0108 (0.011609970230694154) + + + --> TIME: 2024-10-20 08:49:35 -- STEP: 14391/69059 -- GLOBAL_STEP: 83450 + | > loss_text_ce: 0.029985666275024414 (0.03235512884486096) + | > loss_mel_ce: 3.1103789806365967 (3.0993567653050564) + | > loss: 0.7850911617279053 (0.7829279734960608) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2178 (0.24457425730214014) + | > loader_time: 0.011 (0.011610740170966923) + + + --> TIME: 2024-10-20 08:49:54 -- STEP: 14441/69059 -- GLOBAL_STEP: 83500 + | > loss_text_ce: 0.03249984234571457 (0.03235550391053197) + | > loss_mel_ce: 2.9833977222442627 (3.0992995145685587) + | > loss: 0.7539743781089783 (0.7829137545660183) + | > current_lr: 5e-06 + | > step_time: 0.2311 (0.2446132556659579) + | > loader_time: 0.0118 (0.011611145987800294) + + + --> TIME: 2024-10-20 08:50:13 -- STEP: 14491/69059 -- GLOBAL_STEP: 83550 + | > loss_text_ce: 0.03127526491880417 (0.032352718619432) + | > loss_mel_ce: 3.1809396743774414 (3.0992907212886336) + | > loss: 0.8030537366867065 (0.7829108599237368) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2155 (0.24463470202122128) + | > loader_time: 0.0127 (0.011611353264331487) + + + --> TIME: 2024-10-20 08:50:32 -- STEP: 14541/69059 -- GLOBAL_STEP: 83600 + | > loss_text_ce: 0.03456714004278183 (0.03235321649950454) + | > loss_mel_ce: 2.8782706260681152 (3.0994240006606266) + | > loss: 0.7282094359397888 (0.7829443042441098) + | > current_lr: 5e-06 + | > step_time: 0.2448 (0.24468770306553578) + | > loader_time: 0.0102 (0.011612414592283592) + + + --> TIME: 2024-10-20 08:50:50 -- STEP: 14591/69059 -- GLOBAL_STEP: 83650 + | > loss_text_ce: 0.03241175413131714 (0.032353035092673815) + | > loss_mel_ce: 3.037296772003174 (3.0992531459266632) + | > loss: 0.7674271464347839 (0.7829015451995263) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.232 (0.24465564957263744) + | > loader_time: 0.0111 (0.011610830322824096) + + + --> TIME: 2024-10-20 08:51:09 -- STEP: 14641/69059 -- GLOBAL_STEP: 83700 + | > loss_text_ce: 0.03214599937200546 (0.032351997657235704) + | > loss_mel_ce: 3.324596881866455 (3.0992880620703027) + | > loss: 0.8391857147216797 (0.7829100148663651) + | > current_lr: 5e-06 + | > step_time: 0.2237 (0.24465134321165483) + | > loader_time: 0.0106 (0.011609789696753312) + + + --> TIME: 2024-10-20 08:51:27 -- STEP: 14691/69059 -- GLOBAL_STEP: 83750 + | > loss_text_ce: 0.03197131305932999 (0.03234990293439179) + | > loss_mel_ce: 3.1654300689697266 (3.0992273347469137) + | > loss: 0.7993503212928772 (0.7828943093552834) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1944 (0.24462316528121933) + | > loader_time: 0.011 (0.011609047397714565) + + + --> TIME: 2024-10-20 08:51:45 -- STEP: 14741/69059 -- GLOBAL_STEP: 83800 + | > loss_text_ce: 0.03167669475078583 (0.03234991572239338) + | > loss_mel_ce: 3.064204692840576 (3.0990812620292734) + | > loss: 0.773970365524292 (0.7828577943932169) + | > current_lr: 5e-06 + | > step_time: 0.374 (0.2446377118708895) + | > loader_time: 0.0129 (0.011609409114381446) + + + --> TIME: 2024-10-20 08:52:05 -- STEP: 14791/69059 -- GLOBAL_STEP: 83850 + | > loss_text_ce: 0.030868753790855408 (0.03234900241401269) + | > loss_mel_ce: 3.2802186012268066 (3.0990654508737565) + | > loss: 0.8277718424797058 (0.7828536132763232) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3081 (0.2446924931346488) + | > loader_time: 0.0118 (0.011609404011790177) + + + --> TIME: 2024-10-20 08:52:23 -- STEP: 14841/69059 -- GLOBAL_STEP: 83900 + | > loss_text_ce: 0.03359995782375336 (0.032348408502884056) + | > loss_mel_ce: 2.938976764678955 (3.099120223934249) + | > loss: 0.743144154548645 (0.7828671580761489) + | > current_lr: 5e-06 + | > step_time: 0.2616 (0.24467684905987355) + | > loader_time: 0.0116 (0.01160777064155647) + + + --> TIME: 2024-10-20 08:52:41 -- STEP: 14891/69059 -- GLOBAL_STEP: 83950 + | > loss_text_ce: 0.030356410890817642 (0.03234779886682764) + | > loss_mel_ce: 3.482461452484131 (3.0990448050562676) + | > loss: 0.8782044649124146 (0.7828481509449988) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5443 (0.24465773209852418) + | > loader_time: 0.0136 (0.01160651788422714) + + + --> TIME: 2024-10-20 08:53:01 -- STEP: 14941/69059 -- GLOBAL_STEP: 84000 + | > loss_text_ce: 0.032316796481609344 (0.032347395586021135) + | > loss_mel_ce: 3.1009528636932373 (3.099066554267325) + | > loss: 0.7833173871040344 (0.7828534874449476) + | > current_lr: 5e-06 + | > step_time: 0.1934 (0.24471992679121315) + | > loader_time: 0.0109 (0.011608100529456215) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_84000.pth + + --> TIME: 2024-10-20 08:53:35 -- STEP: 14991/69059 -- GLOBAL_STEP: 84050 + | > loss_text_ce: 0.03004259057343006 (0.03234602734207998) + | > loss_mel_ce: 3.629490375518799 (3.099039297219663) + | > loss: 0.9148832559585571 (0.7828463311232264) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3772 (0.2447100401513135) + | > loader_time: 0.0126 (0.011609206308748251) + + + --> TIME: 2024-10-20 08:53:54 -- STEP: 15041/69059 -- GLOBAL_STEP: 84100 + | > loss_text_ce: 0.03173058107495308 (0.03234644796973989) + | > loss_mel_ce: 2.70326566696167 (3.0988081571990977) + | > loss: 0.6837490797042847 (0.7827886512808776) + | > current_lr: 5e-06 + | > step_time: 0.1771 (0.2447213586059453) + | > loader_time: 0.0109 (0.011609223760606538) + + + --> TIME: 2024-10-20 08:54:12 -- STEP: 15091/69059 -- GLOBAL_STEP: 84150 + | > loss_text_ce: 0.03126417472958565 (0.03234598851475095) + | > loss_mel_ce: 3.3887903690338135 (3.0987234891476025) + | > loss: 0.8550136089324951 (0.7827673694002211) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2377 (0.24469061318389246) + | > loader_time: 0.0115 (0.011608986832119061) + + + --> TIME: 2024-10-20 08:54:31 -- STEP: 15141/69059 -- GLOBAL_STEP: 84200 + | > loss_text_ce: 0.031069649383425713 (0.032345034727173406) + | > loss_mel_ce: 2.7944865226745605 (3.0987092543684427) + | > loss: 0.7063890695571899 (0.7827635722654459) + | > current_lr: 5e-06 + | > step_time: 0.2012 (0.24470173708711196) + | > loader_time: 0.0107 (0.011608523096168144) + + + --> TIME: 2024-10-20 08:54:50 -- STEP: 15191/69059 -- GLOBAL_STEP: 84250 + | > loss_text_ce: 0.03374218940734863 (0.032345596834066986) + | > loss_mel_ce: 2.7885751724243164 (3.098788890065814) + | > loss: 0.7055793404579163 (0.7827836217147925) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1949 (0.24474704315032872) + | > loader_time: 0.0101 (0.011609819610110577) + + + --> TIME: 2024-10-20 08:55:08 -- STEP: 15241/69059 -- GLOBAL_STEP: 84300 + | > loss_text_ce: 0.03527163341641426 (0.032345122431561825) + | > loss_mel_ce: 3.2653846740722656 (3.0985749364016546) + | > loss: 0.8251640796661377 (0.7827300146866413) + | > current_lr: 5e-06 + | > step_time: 0.2659 (0.24472490970438124) + | > loader_time: 0.0126 (0.011608882976887678) + + + --> TIME: 2024-10-20 08:55:27 -- STEP: 15291/69059 -- GLOBAL_STEP: 84350 + | > loss_text_ce: 0.03216344118118286 (0.032344598855917894) + | > loss_mel_ce: 2.905801773071289 (3.098555728488123) + | > loss: 0.7344912886619568 (0.7827250818173723) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1981 (0.24474002691132588) + | > loader_time: 0.0103 (0.011610045067254137) + + + --> TIME: 2024-10-20 08:55:44 -- STEP: 15341/69059 -- GLOBAL_STEP: 84400 + | > loss_text_ce: 0.031098071485757828 (0.032342904097577035) + | > loss_mel_ce: 3.0394155979156494 (3.0983487169556545) + | > loss: 0.7676284313201904 (0.7826729052637627) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.2447018937907298) + | > loader_time: 0.0116 (0.011609661183606861) + + + --> TIME: 2024-10-20 08:56:03 -- STEP: 15391/69059 -- GLOBAL_STEP: 84450 + | > loss_text_ce: 0.03477608785033226 (0.03234165889200682) + | > loss_mel_ce: 2.842094659805298 (3.0982077621002415) + | > loss: 0.7192176580429077 (0.7826373552487574) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.308 (0.24468837325920428) + | > loader_time: 0.0123 (0.011609097328282728) + + + --> TIME: 2024-10-20 08:56:23 -- STEP: 15441/69059 -- GLOBAL_STEP: 84500 + | > loss_text_ce: 0.030917411670088768 (0.032341928430397474) + | > loss_mel_ce: 2.8925914764404297 (3.0981892061915985) + | > loss: 0.7308772206306458 (0.7826327836681043) + | > current_lr: 5e-06 + | > step_time: 0.1671 (0.2447683498978453) + | > loader_time: 0.0101 (0.011610146855359958) + + + --> TIME: 2024-10-20 08:56:40 -- STEP: 15491/69059 -- GLOBAL_STEP: 84550 + | > loss_text_ce: 0.03353366255760193 (0.03234069089230141) + | > loss_mel_ce: 3.479405641555786 (3.0981107702495954) + | > loss: 0.8782348036766052 (0.7826128652855938) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4852 (0.24471643244840804) + | > loader_time: 0.0147 (0.011610100832774127) + + + --> TIME: 2024-10-20 08:56:59 -- STEP: 15541/69059 -- GLOBAL_STEP: 84600 + | > loss_text_ce: 0.033364079892635345 (0.032340078655582485) + | > loss_mel_ce: 3.176482915878296 (3.0981358619632493) + | > loss: 0.8024617433547974 (0.7826189851562955) + | > current_lr: 5e-06 + | > step_time: 0.4548 (0.244739062675185) + | > loader_time: 0.0137 (0.011609650373106042) + + + --> TIME: 2024-10-20 08:57:17 -- STEP: 15591/69059 -- GLOBAL_STEP: 84650 + | > loss_text_ce: 0.02980766072869301 (0.03233883728244393) + | > loss_mel_ce: 2.8409228324890137 (3.098143054235356) + | > loss: 0.7176826000213623 (0.7826204728792703) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2088 (0.24469889666865788) + | > loader_time: 0.0109 (0.01160841532250841) + + + --> TIME: 2024-10-20 08:57:35 -- STEP: 15641/69059 -- GLOBAL_STEP: 84700 + | > loss_text_ce: 0.03110997937619686 (0.032337564169635874) + | > loss_mel_ce: 3.2274324893951416 (3.098053117518183) + | > loss: 0.8146356344223022 (0.782597670423413) + | > current_lr: 5e-06 + | > step_time: 0.1862 (0.2446627792026463) + | > loader_time: 0.0103 (0.01160719068327356) + + + --> TIME: 2024-10-20 08:57:53 -- STEP: 15691/69059 -- GLOBAL_STEP: 84750 + | > loss_text_ce: 0.033526912331581116 (0.03233585854194939) + | > loss_mel_ce: 3.1631972789764404 (3.0979832902889717) + | > loss: 0.7991810441017151 (0.7825797872070471) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.305 (0.24464916465736913) + | > loader_time: 0.012 (0.011606157901037764) + + + --> TIME: 2024-10-20 08:58:12 -- STEP: 15741/69059 -- GLOBAL_STEP: 84800 + | > loss_text_ce: 0.03325395658612251 (0.032335275278389315) + | > loss_mel_ce: 2.8779056072235107 (3.0980743077316113) + | > loss: 0.7277898788452148 (0.7826023957461394) + | > current_lr: 5e-06 + | > step_time: 0.1709 (0.24468270100008982) + | > loader_time: 0.01 (0.0116061578597572) + + + --> TIME: 2024-10-20 08:58:30 -- STEP: 15791/69059 -- GLOBAL_STEP: 84850 + | > loss_text_ce: 0.03198545053601265 (0.03233466874783331) + | > loss_mel_ce: 3.2214438915252686 (3.098016271672113) + | > loss: 0.8133573532104492 (0.7825877350992652) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2144 (0.2446722822912242) + | > loader_time: 0.0108 (0.011605049401085745) + + + --> TIME: 2024-10-20 08:58:49 -- STEP: 15841/69059 -- GLOBAL_STEP: 84900 + | > loss_text_ce: 0.035139091312885284 (0.03233532972547326) + | > loss_mel_ce: 3.0283877849578857 (3.097976716839072) + | > loss: 0.7658817172050476 (0.7825780116347568) + | > current_lr: 5e-06 + | > step_time: 0.4922 (0.24469231367186461) + | > loader_time: 0.0136 (0.011604849960036277) + + + --> TIME: 2024-10-20 08:59:08 -- STEP: 15891/69059 -- GLOBAL_STEP: 84950 + | > loss_text_ce: 0.031175652518868446 (0.03233497967641987) + | > loss_mel_ce: 2.9342498779296875 (3.0980263748370582) + | > loss: 0.741356372833252 (0.7825903386115781) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.19 (0.24470485966378588) + | > loader_time: 0.0114 (0.011603965714867392) + + + --> TIME: 2024-10-20 08:59:26 -- STEP: 15941/69059 -- GLOBAL_STEP: 85000 + | > loss_text_ce: 0.030559062957763672 (0.03233453245376326) + | > loss_mel_ce: 2.845522880554199 (3.0978658396135366) + | > loss: 0.7190204858779907 (0.782550093012063) + | > current_lr: 5e-06 + | > step_time: 0.2016 (0.2446855964329213) + | > loader_time: 0.0117 (0.01160272529826518) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_85000.pth + + --> TIME: 2024-10-20 09:00:05 -- STEP: 15991/69059 -- GLOBAL_STEP: 85050 + | > loss_text_ce: 0.03456205502152443 (0.03233323152590048) + | > loss_mel_ce: 2.987438440322876 (3.09774284781453) + | > loss: 0.7555001378059387 (0.7825190198256721) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3555 (0.24477374789996756) + | > loader_time: 0.0115 (0.011604473042328594) + + + --> TIME: 2024-10-20 09:00:23 -- STEP: 16041/69059 -- GLOBAL_STEP: 85100 + | > loss_text_ce: 0.033127523958683014 (0.03233319708201862) + | > loss_mel_ce: 3.2465593814849854 (3.0977349293052856) + | > loss: 0.8199217319488525 (0.78251703157366) + | > current_lr: 5e-06 + | > step_time: 0.2615 (0.24475636374332851) + | > loader_time: 0.0122 (0.011604372978388835) + + + --> TIME: 2024-10-20 09:00:41 -- STEP: 16091/69059 -- GLOBAL_STEP: 85150 + | > loss_text_ce: 0.030539676547050476 (0.032331333685344577) + | > loss_mel_ce: 2.9847989082336426 (3.097653294445291) + | > loss: 0.7538346648216248 (0.7824961570124008) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2177 (0.24473851154252244) + | > loader_time: 0.0111 (0.011605186464180399) + + + --> TIME: 2024-10-20 09:00:59 -- STEP: 16141/69059 -- GLOBAL_STEP: 85200 + | > loss_text_ce: 0.033662185072898865 (0.03233083282673497) + | > loss_mel_ce: 2.8505730628967285 (3.0976547519555497) + | > loss: 0.7210587859153748 (0.7824963961796169) + | > current_lr: 5e-06 + | > step_time: 0.2112 (0.24470011238626121) + | > loader_time: 0.0112 (0.011605662326447634) + + + --> TIME: 2024-10-20 09:01:18 -- STEP: 16191/69059 -- GLOBAL_STEP: 85250 + | > loss_text_ce: 0.0345785953104496 (0.03232950981784408) + | > loss_mel_ce: 3.1063342094421387 (3.0975367633137965) + | > loss: 0.7852281928062439 (0.7824665682759495) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2481 (0.24471678778590833) + | > loader_time: 0.0113 (0.011607259222962137) + + + --> TIME: 2024-10-20 09:01:36 -- STEP: 16241/69059 -- GLOBAL_STEP: 85300 + | > loss_text_ce: 0.03272968903183937 (0.032328644725886896) + | > loss_mel_ce: 3.4762792587280273 (3.0974726293938586) + | > loss: 0.8772522211074829 (0.7824503185236567) + | > current_lr: 5e-06 + | > step_time: 0.2057 (0.24470873288895126) + | > loader_time: 0.0116 (0.011608537639070441) + + + --> TIME: 2024-10-20 09:01:53 -- STEP: 16291/69059 -- GLOBAL_STEP: 85350 + | > loss_text_ce: 0.03325435519218445 (0.03232707264676255) + | > loss_mel_ce: 3.307844877243042 (3.0973254185736803) + | > loss: 0.8352748155593872 (0.7824131228136282) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2075 (0.24465421516294825) + | > loader_time: 0.0109 (0.011616963106773254) + + + --> TIME: 2024-10-20 09:02:12 -- STEP: 16341/69059 -- GLOBAL_STEP: 85400 + | > loss_text_ce: 0.03280226141214371 (0.03232659101721046) + | > loss_mel_ce: 2.8414807319641113 (3.09718419821097) + | > loss: 0.7185707688331604 (0.7823776973080989) + | > current_lr: 5e-06 + | > step_time: 0.178 (0.244643239144501) + | > loader_time: 0.0108 (0.011617661791809009) + + + --> TIME: 2024-10-20 09:02:30 -- STEP: 16391/69059 -- GLOBAL_STEP: 85450 + | > loss_text_ce: 0.03487454727292061 (0.032325406246022564) + | > loss_mel_ce: 3.3832876682281494 (3.0971688772162778) + | > loss: 0.8545405268669128 (0.7823735708635007) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.229 (0.24462281483057968) + | > loader_time: 0.0108 (0.011618805909111735) + + + --> TIME: 2024-10-20 09:02:49 -- STEP: 16441/69059 -- GLOBAL_STEP: 85500 + | > loss_text_ce: 0.03067370317876339 (0.0323261152895133) + | > loss_mel_ce: 3.0883588790893555 (3.0969595060782966) + | > loss: 0.7797581553459167 (0.7823214053302828) + | > current_lr: 5e-06 + | > step_time: 0.2356 (0.2446324060769653) + | > loader_time: 0.0115 (0.011618892537888451) + + + --> TIME: 2024-10-20 09:03:07 -- STEP: 16491/69059 -- GLOBAL_STEP: 85550 + | > loss_text_ce: 0.028055408969521523 (0.032325515892558186) + | > loss_mel_ce: 3.106330156326294 (3.096909504157677) + | > loss: 0.783596396446228 (0.7823087550027883) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2474 (0.24461722449141388) + | > loader_time: 0.011 (0.011619452818334545) + + + --> TIME: 2024-10-20 09:03:25 -- STEP: 16541/69059 -- GLOBAL_STEP: 85600 + | > loss_text_ce: 0.03259052336215973 (0.03232529174330872) + | > loss_mel_ce: 3.2377021312713623 (3.0968914208551395) + | > loss: 0.8175731897354126 (0.7823041781434463) + | > current_lr: 5e-06 + | > step_time: 0.1733 (0.24460004383675474) + | > loader_time: 0.0102 (0.01162007143142746) + + + --> TIME: 2024-10-20 09:03:44 -- STEP: 16591/69059 -- GLOBAL_STEP: 85650 + | > loss_text_ce: 0.03261016681790352 (0.03232469103503379) + | > loss_mel_ce: 3.126082181930542 (3.096831385491948) + | > loss: 0.7896730899810791 (0.7822890191251775) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2655 (0.24459579815651022) + | > loader_time: 0.0156 (0.01162029765526116) + + + --> TIME: 2024-10-20 09:04:02 -- STEP: 16641/69059 -- GLOBAL_STEP: 85700 + | > loss_text_ce: 0.03305548056960106 (0.03232317749064131) + | > loss_mel_ce: 3.052863359451294 (3.096799125823826) + | > loss: 0.7714797258377075 (0.7822805758236359) + | > current_lr: 5e-06 + | > step_time: 0.1796 (0.24458800135277178) + | > loader_time: 0.0121 (0.011621364986414426) + + + --> TIME: 2024-10-20 09:04:19 -- STEP: 16691/69059 -- GLOBAL_STEP: 85750 + | > loss_text_ce: 0.030748803168535233 (0.03232122550002457) + | > loss_mel_ce: 3.148136615753174 (3.0966966223183925) + | > loss: 0.7947213649749756 (0.7822544619375857) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1963 (0.24452310845190686) + | > loader_time: 0.0105 (0.011621363159709023) + + + --> TIME: 2024-10-20 09:04:37 -- STEP: 16741/69059 -- GLOBAL_STEP: 85800 + | > loss_text_ce: 0.029939820989966393 (0.03231955403890994) + | > loss_mel_ce: 3.0687143802642822 (3.0966089911717503) + | > loss: 0.7746635675430298 (0.7822321362873106) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.24451108142884007) + | > loader_time: 0.0118 (0.011622093404737218) + + + --> TIME: 2024-10-20 09:04:56 -- STEP: 16791/69059 -- GLOBAL_STEP: 85850 + | > loss_text_ce: 0.034947022795677185 (0.032319953062670395) + | > loss_mel_ce: 3.3456063270568848 (3.0967475755397422) + | > loss: 0.8451383113861084 (0.7822668821437808) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.307 (0.24452969109378073) + | > loader_time: 0.0118 (0.01162461718782535) + + + --> TIME: 2024-10-20 09:05:15 -- STEP: 16841/69059 -- GLOBAL_STEP: 85900 + | > loss_text_ce: 0.031146647408604622 (0.03231915118748252) + | > loss_mel_ce: 3.100811243057251 (3.096684476923088) + | > loss: 0.782989501953125 (0.7822509070316517) + | > current_lr: 5e-06 + | > step_time: 0.2108 (0.24453059053146384) + | > loader_time: 0.0118 (0.01162502600159813) + + + --> TIME: 2024-10-20 09:05:33 -- STEP: 16891/69059 -- GLOBAL_STEP: 85950 + | > loss_text_ce: 0.031053248792886734 (0.032317601443157734) + | > loss_mel_ce: 3.032130241394043 (3.09679934556584) + | > loss: 0.765795886516571 (0.7822792367559431) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.234 (0.2445189562990586) + | > loader_time: 0.0116 (0.011625185535641635) + + + --> TIME: 2024-10-20 09:05:52 -- STEP: 16941/69059 -- GLOBAL_STEP: 86000 + | > loss_text_ce: 0.035879701375961304 (0.03231682476498725) + | > loss_mel_ce: 3.062979221343994 (3.09680555055516) + | > loss: 0.7747147083282471 (0.7822805938237418) + | > current_lr: 5e-06 + | > step_time: 0.2346 (0.24454765064813033) + | > loader_time: 0.011 (0.011626067053757325) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_86000.pth + + --> TIME: 2024-10-20 09:06:26 -- STEP: 16991/69059 -- GLOBAL_STEP: 86050 + | > loss_text_ce: 0.03331451490521431 (0.0323173898742457) + | > loss_mel_ce: 2.89335298538208 (3.096755103936969) + | > loss: 0.7316668629646301 (0.7822681234426082) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1978 (0.24459234109642475) + | > loader_time: 0.0105 (0.011627582487522022) + + + --> TIME: 2024-10-20 09:06:45 -- STEP: 17041/69059 -- GLOBAL_STEP: 86100 + | > loss_text_ce: 0.03220989182591438 (0.032317507678789485) + | > loss_mel_ce: 3.221388101577759 (3.0968426344897093) + | > loss: 0.8133994936943054 (0.7822900355180502) + | > current_lr: 5e-06 + | > step_time: 0.2572 (0.24463876560885664) + | > loader_time: 0.0124 (0.011628697451679984) + + + --> TIME: 2024-10-20 09:07:05 -- STEP: 17091/69059 -- GLOBAL_STEP: 86150 + | > loss_text_ce: 0.03225510194897652 (0.03231772612419621) + | > loss_mel_ce: 3.1768765449523926 (3.096904164332627) + | > loss: 0.8022829294204712 (0.7823054725978307) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2168 (0.2446940247516344) + | > loader_time: 0.0108 (0.011628957177246387) + + + --> TIME: 2024-10-20 09:07:24 -- STEP: 17141/69059 -- GLOBAL_STEP: 86200 + | > loss_text_ce: 0.034900255501270294 (0.0323181020111269) + | > loss_mel_ce: 3.2409048080444336 (3.096873401587097) + | > loss: 0.8189512491226196 (0.7822978758775782) + | > current_lr: 5e-06 + | > step_time: 0.256 (0.24470024831889922) + | > loader_time: 0.0124 (0.011629046584836444) + + + --> TIME: 2024-10-20 09:07:42 -- STEP: 17191/69059 -- GLOBAL_STEP: 86250 + | > loss_text_ce: 0.029466520994901657 (0.0323157821684475) + | > loss_mel_ce: 2.768279790878296 (3.0968245796616105) + | > loss: 0.699436604976654 (0.7822850904403137) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1766 (0.2446799144009273) + | > loader_time: 0.0131 (0.01162837992773501) + + + --> TIME: 2024-10-20 09:08:00 -- STEP: 17241/69059 -- GLOBAL_STEP: 86300 + | > loss_text_ce: 0.030661363154649734 (0.03231544656138124) + | > loss_mel_ce: 2.7038300037384033 (3.0967801195529283) + | > loss: 0.6836228370666504 (0.7822738915126961) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24468230973000285) + | > loader_time: 0.0107 (0.011628230620229852) + + + --> TIME: 2024-10-20 09:08:18 -- STEP: 17291/69059 -- GLOBAL_STEP: 86350 + | > loss_text_ce: 0.034787584096193314 (0.03231470313475407) + | > loss_mel_ce: 3.2730841636657715 (3.0966480734252975) + | > loss: 0.8269679546356201 (0.7822406941197614) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3336 (0.24465628686783247) + | > loader_time: 0.0121 (0.011627629400161429) + + + --> TIME: 2024-10-20 09:08:36 -- STEP: 17341/69059 -- GLOBAL_STEP: 86400 + | > loss_text_ce: 0.032540883868932724 (0.03231369114763056) + | > loss_mel_ce: 3.543558359146118 (3.0966981958953825) + | > loss: 0.8940247893333435 (0.782252971729765) + | > current_lr: 5e-06 + | > step_time: 0.4062 (0.2446201756286358) + | > loader_time: 0.0129 (0.011626931981818791) + + + --> TIME: 2024-10-20 09:08:55 -- STEP: 17391/69059 -- GLOBAL_STEP: 86450 + | > loss_text_ce: 0.029295844957232475 (0.03231251181736993) + | > loss_mel_ce: 3.3211021423339844 (3.0967501344527752) + | > loss: 0.837599515914917 (0.782265661542983) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1971 (0.24464523930045404) + | > loader_time: 0.0108 (0.011626750136537705) + + + --> TIME: 2024-10-20 09:09:14 -- STEP: 17441/69059 -- GLOBAL_STEP: 86500 + | > loss_text_ce: 0.03400715813040733 (0.03231199828976328) + | > loss_mel_ce: 3.0222434997558594 (3.096706739007025) + | > loss: 0.7640626430511475 (0.7822546842933331) + | > current_lr: 5e-06 + | > step_time: 0.185 (0.2446557571223472) + | > loader_time: 0.0104 (0.011626289153439464) + + + --> TIME: 2024-10-20 09:09:33 -- STEP: 17491/69059 -- GLOBAL_STEP: 86550 + | > loss_text_ce: 0.03462134301662445 (0.032311564838518984) + | > loss_mel_ce: 3.1118428707122803 (3.0967670926834496) + | > loss: 0.7866160273551941 (0.7822696643664626) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5581 (0.24470272653610364) + | > loader_time: 0.0138 (0.011625877750805239) + + + --> TIME: 2024-10-20 09:09:51 -- STEP: 17541/69059 -- GLOBAL_STEP: 86600 + | > loss_text_ce: 0.03213008865714073 (0.032309624383080895) + | > loss_mel_ce: 2.8727967739105225 (3.096513421646534) + | > loss: 0.7262316942214966 (0.7822057614949809) + | > current_lr: 5e-06 + | > step_time: 0.2162 (0.244657781089999) + | > loader_time: 0.0115 (0.01162503539180481) + + + --> TIME: 2024-10-20 09:10:09 -- STEP: 17591/69059 -- GLOBAL_STEP: 86650 + | > loss_text_ce: 0.03055761754512787 (0.03230902420871607) + | > loss_mel_ce: 2.9018094539642334 (3.09648166888203) + | > loss: 0.7330917716026306 (0.7821976732458195) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2043 (0.24465109686509695) + | > loader_time: 0.0109 (0.011625328937348604) + + + --> TIME: 2024-10-20 09:10:27 -- STEP: 17641/69059 -- GLOBAL_STEP: 86700 + | > loss_text_ce: 0.03247281163930893 (0.03230670559195809) + | > loss_mel_ce: 3.2123961448669434 (3.0963841266057885) + | > loss: 0.8112172484397888 (0.7821727080258922) + | > current_lr: 5e-06 + | > step_time: 0.3049 (0.2446087815919558) + | > loader_time: 0.012 (0.01162402051814622) + + + --> TIME: 2024-10-20 09:10:45 -- STEP: 17691/69059 -- GLOBAL_STEP: 86750 + | > loss_text_ce: 0.032282061874866486 (0.032306349382552696) + | > loss_mel_ce: 3.3070006370544434 (3.0963604675019245) + | > loss: 0.8348206877708435 (0.7821667041943519) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2382 (0.24459476999508503) + | > loader_time: 0.0104 (0.011623514735619884) + + + --> TIME: 2024-10-20 09:11:03 -- STEP: 17741/69059 -- GLOBAL_STEP: 86800 + | > loss_text_ce: 0.03176430985331535 (0.03230617446096822) + | > loss_mel_ce: 3.310483455657959 (3.096267803109992) + | > loss: 0.8355619311332703 (0.7821434943680425) + | > current_lr: 5e-06 + | > step_time: 0.184 (0.2445630441287965) + | > loader_time: 0.0103 (0.01162374027014171) + + + --> TIME: 2024-10-20 09:11:21 -- STEP: 17791/69059 -- GLOBAL_STEP: 86850 + | > loss_text_ce: 0.03418003022670746 (0.03230558550543868) + | > loss_mel_ce: 3.148472309112549 (3.096234216288559) + | > loss: 0.795663058757782 (0.7821349504290017) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2581 (0.24455677781537372) + | > loader_time: 0.011 (0.01162263035244891) + + + --> TIME: 2024-10-20 09:11:39 -- STEP: 17841/69059 -- GLOBAL_STEP: 86900 + | > loss_text_ce: 0.030194846913218498 (0.032305029465964204) + | > loss_mel_ce: 3.544447898864746 (3.0961791990343235) + | > loss: 0.8936606645584106 (0.7821210571137462) + | > current_lr: 5e-06 + | > step_time: 0.3046 (0.24454680219318542) + | > loader_time: 0.0117 (0.01162197170126074) + + + --> TIME: 2024-10-20 09:11:58 -- STEP: 17891/69059 -- GLOBAL_STEP: 86950 + | > loss_text_ce: 0.03239421546459198 (0.03230434860460529) + | > loss_mel_ce: 3.2201311588287354 (3.0960899729171816) + | > loss: 0.8131313323974609 (0.782098580356894) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1983 (0.24455016708501964) + | > loader_time: 0.0108 (0.011621081990984085) + + + --> TIME: 2024-10-20 09:12:17 -- STEP: 17941/69059 -- GLOBAL_STEP: 87000 + | > loss_text_ce: 0.030429210513830185 (0.032303703384902224) + | > loss_mel_ce: 2.8890538215637207 (3.096018406813377) + | > loss: 0.7298707365989685 (0.7820805275256159) + | > current_lr: 5e-06 + | > step_time: 0.1955 (0.2445975456287761) + | > loader_time: 0.011 (0.011620905452345785) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_87000.pth + + --> TIME: 2024-10-20 09:12:52 -- STEP: 17991/69059 -- GLOBAL_STEP: 87050 + | > loss_text_ce: 0.031310006976127625 (0.03230308106420814) + | > loss_mel_ce: 3.1717212200164795 (3.0958324804703987) + | > loss: 0.8007578253746033 (0.7820338903648377) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2938 (0.2446155774971702) + | > loader_time: 0.0116 (0.011621194129376896) + + + --> TIME: 2024-10-20 09:13:10 -- STEP: 18041/69059 -- GLOBAL_STEP: 87100 + | > loss_text_ce: 0.030310021713376045 (0.032301857485416766) + | > loss_mel_ce: 2.7856600284576416 (3.0957742397605723) + | > loss: 0.703992486000061 (0.782019024300711) + | > current_lr: 5e-06 + | > step_time: 0.205 (0.24458350595273698) + | > loader_time: 0.011 (0.01162811983094885) + + + --> TIME: 2024-10-20 09:13:29 -- STEP: 18091/69059 -- GLOBAL_STEP: 87150 + | > loss_text_ce: 0.031222300603985786 (0.03230212993936758) + | > loss_mel_ce: 3.2757411003112793 (3.0956760666723575) + | > loss: 0.8267408609390259 (0.781994549148507) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1978 (0.24461153648589024) + | > loader_time: 0.0106 (0.011627916778130949) + + + --> TIME: 2024-10-20 09:13:45 -- STEP: 18141/69059 -- GLOBAL_STEP: 87200 + | > loss_text_ce: 0.032675810158252716 (0.032301118564121496) + | > loss_mel_ce: 3.3962554931640625 (3.0954936130406425) + | > loss: 0.8572328090667725 (0.7819486829025541) + | > current_lr: 5e-06 + | > step_time: 0.1843 (0.2445154710638965) + | > loader_time: 0.0101 (0.011627036847942624) + + + --> TIME: 2024-10-20 09:14:04 -- STEP: 18191/69059 -- GLOBAL_STEP: 87250 + | > loss_text_ce: 0.03082013502717018 (0.03230005814172451) + | > loss_mel_ce: 3.1582186222076416 (3.095453884568682) + | > loss: 0.7972596883773804 (0.7819384856795757) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.239 (0.24453256498639508) + | > loader_time: 0.011 (0.011627874158920404) + + + --> TIME: 2024-10-20 09:14:23 -- STEP: 18241/69059 -- GLOBAL_STEP: 87300 + | > loss_text_ce: 0.032793041318655014 (0.03229863784027274) + | > loss_mel_ce: 3.370076894760132 (3.095423260420508) + | > loss: 0.8507174849510193 (0.7819304745661684) + | > current_lr: 5e-06 + | > step_time: 0.3223 (0.24454892652154103) + | > loader_time: 0.0124 (0.011627854841345353) + + + --> TIME: 2024-10-20 09:14:41 -- STEP: 18291/69059 -- GLOBAL_STEP: 87350 + | > loss_text_ce: 0.032054390758275986 (0.03229832708203031) + | > loss_mel_ce: 3.2553749084472656 (3.0954692921365394) + | > loss: 0.8218573331832886 (0.7819419048071149) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2327 (0.2445395924161128) + | > loader_time: 0.0118 (0.011627479233297818) + + + --> TIME: 2024-10-20 09:15:00 -- STEP: 18341/69059 -- GLOBAL_STEP: 87400 + | > loss_text_ce: 0.03150508180260658 (0.03229753073335582) + | > loss_mel_ce: 3.5009944438934326 (3.095520825352494) + | > loss: 0.8831248879432678 (0.7819545890114878) + | > current_lr: 5e-06 + | > step_time: 0.3107 (0.24457157737757318) + | > loader_time: 0.0128 (0.01162834456323) + + + --> TIME: 2024-10-20 09:15:20 -- STEP: 18391/69059 -- GLOBAL_STEP: 87450 + | > loss_text_ce: 0.03403187543153763 (0.03229667208169197) + | > loss_mel_ce: 3.045522689819336 (3.095517619529519) + | > loss: 0.7698886394500732 (0.7819535728995908) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3087 (0.24463146276454822) + | > loader_time: 0.0101 (0.011628237110801921) + + + --> TIME: 2024-10-20 09:15:38 -- STEP: 18441/69059 -- GLOBAL_STEP: 87500 + | > loss_text_ce: 0.03168671205639839 (0.03229626913675537) + | > loss_mel_ce: 3.169408082962036 (3.095438471469291) + | > loss: 0.8002737164497375 (0.7819336851470207) + | > current_lr: 5e-06 + | > step_time: 0.1707 (0.24460697347317162) + | > loader_time: 0.0104 (0.011627322376876581) + + + --> TIME: 2024-10-20 09:15:55 -- STEP: 18491/69059 -- GLOBAL_STEP: 87550 + | > loss_text_ce: 0.03177855536341667 (0.03229417111532236) + | > loss_mel_ce: 2.9958698749542236 (3.0952794728106943) + | > loss: 0.756912112236023 (0.7818934109784614) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2168 (0.24455786095606266) + | > loader_time: 0.0117 (0.011626198991781007) + + + --> TIME: 2024-10-20 09:16:13 -- STEP: 18541/69059 -- GLOBAL_STEP: 87600 + | > loss_text_ce: 0.029351582750678062 (0.032292306358992946) + | > loss_mel_ce: 3.139234781265259 (3.0951165206917857) + | > loss: 0.7921465635299683 (0.7818522067634777) + | > current_lr: 5e-06 + | > step_time: 0.1772 (0.2445314306251235) + | > loader_time: 0.0108 (0.01162514404458181) + + + --> TIME: 2024-10-20 09:16:32 -- STEP: 18591/69059 -- GLOBAL_STEP: 87650 + | > loss_text_ce: 0.03331700339913368 (0.03229157386832492) + | > loss_mel_ce: 2.767803192138672 (3.09499046076786) + | > loss: 0.7002800703048706 (0.7818205086612043) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1989 (0.24453168778170453) + | > loader_time: 0.0115 (0.011623768441987857) + + + --> TIME: 2024-10-20 09:16:51 -- STEP: 18641/69059 -- GLOBAL_STEP: 87700 + | > loss_text_ce: 0.03307226672768593 (0.03229078975250895) + | > loss_mel_ce: 3.2306742668151855 (3.0949540112920255) + | > loss: 0.8159366250038147 (0.7818112002697559) + | > current_lr: 5e-06 + | > step_time: 0.1725 (0.24453792983348221) + | > loader_time: 0.0104 (0.011623685410219603) + + + --> TIME: 2024-10-20 09:17:10 -- STEP: 18691/69059 -- GLOBAL_STEP: 87750 + | > loss_text_ce: 0.03228829428553581 (0.03229082950433398) + | > loss_mel_ce: 2.7908060550689697 (3.095016143785139) + | > loss: 0.7057735919952393 (0.7818267433252123) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.204 (0.24457983636185135) + | > loader_time: 0.0105 (0.011624093984602166) + + + --> TIME: 2024-10-20 09:17:28 -- STEP: 18741/69059 -- GLOBAL_STEP: 87800 + | > loss_text_ce: 0.03382791206240654 (0.03229015270218955) + | > loss_mel_ce: 2.883192300796509 (3.0949851409277285) + | > loss: 0.7292550802230835 (0.7818188234121299) + | > current_lr: 5e-06 + | > step_time: 0.205 (0.24455916406403402) + | > loader_time: 0.0106 (0.011623654559673688) + + + --> TIME: 2024-10-20 09:17:45 -- STEP: 18791/69059 -- GLOBAL_STEP: 87850 + | > loss_text_ce: 0.0306619331240654 (0.032289425919037724) + | > loss_mel_ce: 3.212625741958618 (3.094833413673151) + | > loss: 0.8108218908309937 (0.7817807099017438) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2172 (0.2445075852983484) + | > loader_time: 0.0109 (0.011622075484905514) + + + --> TIME: 2024-10-20 09:18:04 -- STEP: 18841/69059 -- GLOBAL_STEP: 87900 + | > loss_text_ce: 0.032330069690942764 (0.03228846085965451) + | > loss_mel_ce: 3.317188262939453 (3.0947285327908185) + | > loss: 0.8373795747756958 (0.781754248429874) + | > current_lr: 5e-06 + | > step_time: 0.1869 (0.24450405424399582) + | > loader_time: 0.0109 (0.011621296339357485) + + + --> TIME: 2024-10-20 09:18:22 -- STEP: 18891/69059 -- GLOBAL_STEP: 87950 + | > loss_text_ce: 0.03252255544066429 (0.03228830764006383) + | > loss_mel_ce: 3.3935701847076416 (3.0947261808232795) + | > loss: 0.8565231561660767 (0.7817536221275734) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4355 (0.24451274132502676) + | > loader_time: 0.0127 (0.011621178228248975) + + + --> TIME: 2024-10-20 09:18:41 -- STEP: 18941/69059 -- GLOBAL_STEP: 88000 + | > loss_text_ce: 0.03183458372950554 (0.032287615579280546) + | > loss_mel_ce: 3.3684701919555664 (3.0948138109236947) + | > loss: 0.8500761985778809 (0.7817753566449975) + | > current_lr: 5e-06 + | > step_time: 0.3795 (0.24451442842567614) + | > loader_time: 0.0121 (0.011621190290580466) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_88000.pth + + --> TIME: 2024-10-20 09:19:15 -- STEP: 18991/69059 -- GLOBAL_STEP: 88050 + | > loss_text_ce: 0.032494619488716125 (0.032286713904343955) + | > loss_mel_ce: 3.262834072113037 (3.094884020866261) + | > loss: 0.8238321542739868 (0.7817926837064594) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2147 (0.24456772385700135) + | > loader_time: 0.0104 (0.011621437782824734) + + + --> TIME: 2024-10-20 09:19:34 -- STEP: 19041/69059 -- GLOBAL_STEP: 88100 + | > loss_text_ce: 0.03497214987874031 (0.03228665817133583) + | > loss_mel_ce: 3.2956624031066895 (3.0948625154827196) + | > loss: 0.8326586484909058 (0.7817872934284839) + | > current_lr: 5e-06 + | > step_time: 0.406 (0.24459608319514287) + | > loader_time: 0.0132 (0.011621480741631588) + + + --> TIME: 2024-10-20 09:19:53 -- STEP: 19091/69059 -- GLOBAL_STEP: 88150 + | > loss_text_ce: 0.03256205841898918 (0.03228634274827889) + | > loss_mel_ce: 3.1924514770507812 (3.094844733767673) + | > loss: 0.8062533736228943 (0.7817827691346748) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3433 (0.2446195374776018) + | > loader_time: 0.0118 (0.011621742436862236) + + + --> TIME: 2024-10-20 09:20:11 -- STEP: 19141/69059 -- GLOBAL_STEP: 88200 + | > loss_text_ce: 0.031390320509672165 (0.03228583734467563) + | > loss_mel_ce: 3.2439017295837402 (3.094729044041175) + | > loss: 0.8188230395317078 (0.7817537203508209) + | > current_lr: 5e-06 + | > step_time: 0.2613 (0.24459163379435028) + | > loader_time: 0.0119 (0.011620775172655742) + + + --> TIME: 2024-10-20 09:20:28 -- STEP: 19191/69059 -- GLOBAL_STEP: 88250 + | > loss_text_ce: 0.032610516995191574 (0.0322844426449833) + | > loss_mel_ce: 2.81343412399292 (3.0946341124565273) + | > loss: 0.7115111351013184 (0.7817296387846009) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2171 (0.24455058156627169) + | > loader_time: 0.0116 (0.011620461366180461) + + + --> TIME: 2024-10-20 09:20:48 -- STEP: 19241/69059 -- GLOBAL_STEP: 88300 + | > loss_text_ce: 0.03198358044028282 (0.03228409476119397) + | > loss_mel_ce: 3.0648553371429443 (3.0944767573613112) + | > loss: 0.77420973777771 (0.7816902130484418) + | > current_lr: 5e-06 + | > step_time: 0.5415 (0.2445843867195923) + | > loader_time: 0.0174 (0.011621028542685907) + + + --> TIME: 2024-10-20 09:21:05 -- STEP: 19291/69059 -- GLOBAL_STEP: 88350 + | > loss_text_ce: 0.029173221439123154 (0.0322836197998243) + | > loss_mel_ce: 3.304173469543457 (3.0943477794347913) + | > loss: 0.8333366513252258 (0.7816578498228263) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1987 (0.24455594632112812) + | > loader_time: 0.011 (0.01162088681735591) + + + --> TIME: 2024-10-20 09:21:23 -- STEP: 19341/69059 -- GLOBAL_STEP: 88400 + | > loss_text_ce: 0.03257978707551956 (0.0322827585424821) + | > loss_mel_ce: 3.342433214187622 (3.0940958011311124) + | > loss: 0.8437532782554626 (0.7815946399422847) + | > current_lr: 5e-06 + | > step_time: 0.3772 (0.24450161368183335) + | > loader_time: 0.0127 (0.011620718655853254) + + + --> TIME: 2024-10-20 09:21:42 -- STEP: 19391/69059 -- GLOBAL_STEP: 88450 + | > loss_text_ce: 0.029176166281104088 (0.03228266453946333) + | > loss_mel_ce: 3.193594217300415 (3.0940226798858874) + | > loss: 0.8056926131248474 (0.7815763361309785) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2115 (0.24452935754837882) + | > loader_time: 0.011 (0.011620880482730562) + + + --> TIME: 2024-10-20 09:22:01 -- STEP: 19441/69059 -- GLOBAL_STEP: 88500 + | > loss_text_ce: 0.032932210713624954 (0.032281718634808394) + | > loss_mel_ce: 3.5207343101501465 (3.0941218851672248) + | > loss: 0.8884166479110718 (0.781600900981218) + | > current_lr: 5e-06 + | > step_time: 0.3758 (0.24455099849996248) + | > loader_time: 0.015 (0.011620923807012622) + + + --> TIME: 2024-10-20 09:22:19 -- STEP: 19491/69059 -- GLOBAL_STEP: 88550 + | > loss_text_ce: 0.03149328753352165 (0.03228034800738778) + | > loss_mel_ce: 2.7247393131256104 (3.0941233012205336) + | > loss: 0.6890581250190735 (0.7816009123324982) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1911 (0.24455569349571465) + | > loader_time: 0.0104 (0.011620797553661107) + + + --> TIME: 2024-10-20 09:22:36 -- STEP: 19541/69059 -- GLOBAL_STEP: 88600 + | > loss_text_ce: 0.03166217356920242 (0.032279430435681065) + | > loss_mel_ce: 2.978367328643799 (3.0941176535522974) + | > loss: 0.7525073885917664 (0.7815992710265941) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.24449879757964288) + | > loader_time: 0.0117 (0.01162036913126094) + + + --> TIME: 2024-10-20 09:22:55 -- STEP: 19591/69059 -- GLOBAL_STEP: 88650 + | > loss_text_ce: 0.03254835680127144 (0.0322780569469668) + | > loss_mel_ce: 3.4420008659362793 (3.094008583745284) + | > loss: 0.8686373233795166 (0.7815716602001385) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5398 (0.24448972568353278) + | > loader_time: 0.0136 (0.011619161351048011) + + + --> TIME: 2024-10-20 09:23:12 -- STEP: 19641/69059 -- GLOBAL_STEP: 88700 + | > loss_text_ce: 0.029801856726408005 (0.03227737762967502) + | > loss_mel_ce: 2.9778530597686768 (3.0939319913434935) + | > loss: 0.7519137263298035 (0.7815523422739731) + | > current_lr: 5e-06 + | > step_time: 0.1813 (0.24443840524110133) + | > loader_time: 0.0138 (0.01161807949386046) + + + --> TIME: 2024-10-20 09:23:32 -- STEP: 19691/69059 -- GLOBAL_STEP: 88750 + | > loss_text_ce: 0.029854165390133858 (0.03227746035650382) + | > loss_mel_ce: 2.938051462173462 (3.0939810344562693) + | > loss: 0.7419763803482056 (0.7815646237394953) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2144 (0.24448722952976926) + | > loader_time: 0.0107 (0.011617699873452878) + + + --> TIME: 2024-10-20 09:23:50 -- STEP: 19741/69059 -- GLOBAL_STEP: 88800 + | > loss_text_ce: 0.03197403997182846 (0.03227649212429301) + | > loss_mel_ce: 3.018789768218994 (3.0938426913609214) + | > loss: 0.7626909613609314 (0.7815297959175866) + | > current_lr: 5e-06 + | > step_time: 0.5867 (0.24449235797200156) + | > loader_time: 0.0141 (0.011616644190954512) + + + --> TIME: 2024-10-20 09:24:09 -- STEP: 19791/69059 -- GLOBAL_STEP: 88850 + | > loss_text_ce: 0.03311290964484215 (0.032275566203196715) + | > loss_mel_ce: 3.1279983520507812 (3.0938432611900324) + | > loss: 0.7902778387069702 (0.7815297068921205) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2832 (0.24450103805688106) + | > loader_time: 0.0109 (0.01161571698518625) + + + --> TIME: 2024-10-20 09:24:28 -- STEP: 19841/69059 -- GLOBAL_STEP: 88900 + | > loss_text_ce: 0.030489448457956314 (0.03227530626378196) + | > loss_mel_ce: 2.9142465591430664 (3.0937748848986293) + | > loss: 0.7361840009689331 (0.7815125478286733) + | > current_lr: 5e-06 + | > step_time: 0.1966 (0.2445120257263277) + | > loader_time: 0.0108 (0.011614414600287547) + + + --> TIME: 2024-10-20 09:24:46 -- STEP: 19891/69059 -- GLOBAL_STEP: 88950 + | > loss_text_ce: 0.0335201695561409 (0.032274364846585926) + | > loss_mel_ce: 3.390216112136841 (3.093733291916994) + | > loss: 0.8559340834617615 (0.7815019142321473) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5362 (0.24450830939572293) + | > loader_time: 0.0135 (0.01161370920587179) + + + --> TIME: 2024-10-20 09:25:04 -- STEP: 19941/69059 -- GLOBAL_STEP: 89000 + | > loss_text_ce: 0.03195292130112648 (0.03227365967959549) + | > loss_mel_ce: 3.273379325866699 (3.0937549837864236) + | > loss: 0.8263330459594727 (0.7815071609055985) + | > current_lr: 5e-06 + | > step_time: 0.2628 (0.24450077756370284) + | > loader_time: 0.0112 (0.01161308089144809) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_89000.pth + + --> TIME: 2024-10-20 09:25:38 -- STEP: 19991/69059 -- GLOBAL_STEP: 89050 + | > loss_text_ce: 0.03169658035039902 (0.032273194257832157) + | > loss_mel_ce: 3.2674522399902344 (3.0936940398761656) + | > loss: 0.8247871994972229 (0.7814918085660661) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2385 (0.24453854083800228) + | > loader_time: 0.0114 (0.01161388713788394) + + + --> TIME: 2024-10-20 09:25:57 -- STEP: 20041/69059 -- GLOBAL_STEP: 89100 + | > loss_text_ce: 0.03207499906420708 (0.03227285041194664) + | > loss_mel_ce: 2.858469009399414 (3.093590810373738) + | > loss: 0.7226359844207764 (0.7814659152337398) + | > current_lr: 5e-06 + | > step_time: 0.1716 (0.24454113914964093) + | > loader_time: 0.0103 (0.011614572763335936) + + + --> TIME: 2024-10-20 09:26:14 -- STEP: 20091/69059 -- GLOBAL_STEP: 89150 + | > loss_text_ce: 0.030116451904177666 (0.03227121489253519) + | > loss_mel_ce: 2.8920705318450928 (3.0935721580625266) + | > loss: 0.730546772480011 (0.7814608432797923) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1896 (0.2445173448911321) + | > loader_time: 0.0101 (0.01161432040763599) + + + --> TIME: 2024-10-20 09:26:33 -- STEP: 20141/69059 -- GLOBAL_STEP: 89200 + | > loss_text_ce: 0.033939335495233536 (0.03227065000930001) + | > loss_mel_ce: 3.2713613510131836 (3.0934723518283014) + | > loss: 0.8263251781463623 (0.7814357505110537) + | > current_lr: 5e-06 + | > step_time: 0.3779 (0.24449676997635203) + | > loader_time: 0.0121 (0.011620596606837605) + + + --> TIME: 2024-10-20 09:26:51 -- STEP: 20191/69059 -- GLOBAL_STEP: 89250 + | > loss_text_ce: 0.03151915222406387 (0.032269594566895245) + | > loss_mel_ce: 2.567887306213379 (3.093420203884195) + | > loss: 0.6498516201972961 (0.7814224496691187) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1987 (0.24447379625426338) + | > loader_time: 0.0109 (0.01162093653142096) + + + --> TIME: 2024-10-20 09:27:08 -- STEP: 20241/69059 -- GLOBAL_STEP: 89300 + | > loss_text_ce: 0.032822877168655396 (0.03226851585726635) + | > loss_mel_ce: 3.4188103675842285 (3.0932878449196135) + | > loss: 0.8629083037376404 (0.7813890902405114) + | > current_lr: 5e-06 + | > step_time: 0.2612 (0.2444220634854528) + | > loader_time: 0.0109 (0.011619579708980818) + + + --> TIME: 2024-10-20 09:27:26 -- STEP: 20291/69059 -- GLOBAL_STEP: 89350 + | > loss_text_ce: 0.03288671746850014 (0.03226757368719031) + | > loss_mel_ce: 3.453723430633545 (3.093236777600418) + | > loss: 0.8716525435447693 (0.7813760878670702) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4452 (0.24441468384906165) + | > loader_time: 0.0138 (0.011619629523170277) + + + --> TIME: 2024-10-20 09:27:45 -- STEP: 20341/69059 -- GLOBAL_STEP: 89400 + | > loss_text_ce: 0.033717453479766846 (0.03226824815779582) + | > loss_mel_ce: 3.0004844665527344 (3.093087727303486) + | > loss: 0.7585504651069641 (0.7813389939044256) + | > current_lr: 5e-06 + | > step_time: 0.1724 (0.24443247463492068) + | > loader_time: 0.0108 (0.01161916545110082) + + + --> TIME: 2024-10-20 09:28:02 -- STEP: 20391/69059 -- GLOBAL_STEP: 89450 + | > loss_text_ce: 0.03181801736354828 (0.03226744993256184) + | > loss_mel_ce: 2.9488942623138428 (3.0930350647264353) + | > loss: 0.7451780438423157 (0.7813256287035747) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1895 (0.24440200208045004) + | > loader_time: 0.0105 (0.011618380291442888) + + + --> TIME: 2024-10-20 09:28:20 -- STEP: 20441/69059 -- GLOBAL_STEP: 89500 + | > loss_text_ce: 0.03315151110291481 (0.03226699276478876) + | > loss_mel_ce: 3.2495150566101074 (3.093083323240791) + | > loss: 0.8206666707992554 (0.7813375790486228) + | > current_lr: 5e-06 + | > step_time: 0.2683 (0.24438124995112434) + | > loader_time: 0.0116 (0.011617574735628137) + + + --> TIME: 2024-10-20 09:28:39 -- STEP: 20491/69059 -- GLOBAL_STEP: 89550 + | > loss_text_ce: 0.031229261308908463 (0.032266446438571614) + | > loss_mel_ce: 2.9416987895965576 (3.093107529588477) + | > loss: 0.743232011795044 (0.7813434940522157) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2136 (0.2443787947469445) + | > loader_time: 0.0103 (0.011617691635207043) + + + --> TIME: 2024-10-20 09:28:58 -- STEP: 20541/69059 -- GLOBAL_STEP: 89600 + | > loss_text_ce: 0.030594343319535255 (0.032266651605174) + | > loss_mel_ce: 2.904451608657837 (3.0930367286351053) + | > loss: 0.7337614893913269 (0.7813258451041429) + | > current_lr: 5e-06 + | > step_time: 0.1936 (0.2444230761629461) + | > loader_time: 0.0101 (0.011617728550857416) + + + --> TIME: 2024-10-20 09:29:17 -- STEP: 20591/69059 -- GLOBAL_STEP: 89650 + | > loss_text_ce: 0.030600901693105698 (0.03226484796359334) + | > loss_mel_ce: 3.3000926971435547 (3.0929686702199017) + | > loss: 0.8326733708381653 (0.7813083796015775) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4487 (0.24443334790016247) + | > loader_time: 0.0141 (0.011617484177700612) + + + --> TIME: 2024-10-20 09:29:35 -- STEP: 20641/69059 -- GLOBAL_STEP: 89700 + | > loss_text_ce: 0.03032681718468666 (0.03226499125131526) + | > loss_mel_ce: 2.8777151107788086 (3.092803683173968) + | > loss: 0.7270104885101318 (0.7812671686615299) + | > current_lr: 5e-06 + | > step_time: 0.1712 (0.24441607781236457) + | > loader_time: 0.0101 (0.011616580748845654) + + + --> TIME: 2024-10-20 09:29:53 -- STEP: 20691/69059 -- GLOBAL_STEP: 89750 + | > loss_text_ce: 0.033226530998945236 (0.032265217855593026) + | > loss_mel_ce: 3.2539377212524414 (3.09275201585971) + | > loss: 0.8217910528182983 (0.7812543084844861) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2003 (0.24439287386165948) + | > loader_time: 0.0107 (0.011615137486741044) + + + --> TIME: 2024-10-20 09:30:11 -- STEP: 20741/69059 -- GLOBAL_STEP: 89800 + | > loss_text_ce: 0.031632617115974426 (0.032265019436799824) + | > loss_mel_ce: 2.7389371395111084 (3.092690213874589) + | > loss: 0.6926424503326416 (0.7812388083909609) + | > current_lr: 5e-06 + | > step_time: 0.171 (0.24437313709532163) + | > loader_time: 0.0101 (0.011613960568662669) + + + --> TIME: 2024-10-20 09:30:30 -- STEP: 20791/69059 -- GLOBAL_STEP: 89850 + | > loss_text_ce: 0.03394273295998573 (0.03226438021216294) + | > loss_mel_ce: 3.046496868133545 (3.0926966165362013) + | > loss: 0.7701098918914795 (0.7812402492552046) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2063 (0.2443976117547334) + | > loader_time: 0.0109 (0.01161426584141852) + + + --> TIME: 2024-10-20 09:30:47 -- STEP: 20841/69059 -- GLOBAL_STEP: 89900 + | > loss_text_ce: 0.032978352159261703 (0.0322630790235055) + | > loss_mel_ce: 2.916599750518799 (3.0925378918756303) + | > loss: 0.7373945116996765 (0.7812002427912816) + | > current_lr: 5e-06 + | > step_time: 0.1958 (0.24434508427818127) + | > loader_time: 0.0105 (0.011612833018129256) + + + --> TIME: 2024-10-20 09:31:07 -- STEP: 20891/69059 -- GLOBAL_STEP: 89950 + | > loss_text_ce: 0.03150983527302742 (0.032262306448301865) + | > loss_mel_ce: 3.195812225341797 (3.092494058595299) + | > loss: 0.8068305253982544 (0.7811890913256561) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2156 (0.24439128777858812) + | > loader_time: 0.0106 (0.011612260982456265) + + + --> TIME: 2024-10-20 09:31:25 -- STEP: 20941/69059 -- GLOBAL_STEP: 90000 + | > loss_text_ce: 0.031525611877441406 (0.03226122806674789) + | > loss_mel_ce: 3.2019455432891846 (3.0924494404499114) + | > loss: 0.8083677887916565 (0.7811776672004599) + | > current_lr: 5e-06 + | > step_time: 0.1866 (0.24438561011474205) + | > loader_time: 0.011 (0.01161290980940601) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_90000.pth + + --> TIME: 2024-10-20 09:32:01 -- STEP: 20991/69059 -- GLOBAL_STEP: 90050 + | > loss_text_ce: 0.03286361321806908 (0.03226126726670809) + | > loss_mel_ce: 3.2032370567321777 (3.0924088823398184) + | > loss: 0.8090251684188843 (0.7811675374666119) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2275 (0.24439905166944012) + | > loader_time: 0.0112 (0.011614182662000696) + + + --> TIME: 2024-10-20 09:32:20 -- STEP: 21041/69059 -- GLOBAL_STEP: 90100 + | > loss_text_ce: 0.03158189356327057 (0.03226048853373196) + | > loss_mel_ce: 3.0110154151916504 (3.0924369553457525) + | > loss: 0.7606493234634399 (0.781174361040075) + | > current_lr: 5e-06 + | > step_time: 0.1768 (0.24442319306610696) + | > loader_time: 0.011 (0.011615251850069976) + + + --> TIME: 2024-10-20 09:32:39 -- STEP: 21091/69059 -- GLOBAL_STEP: 90150 + | > loss_text_ce: 0.030694179236888885 (0.032259315639740826) + | > loss_mel_ce: 3.0298116207122803 (3.0924574957933295) + | > loss: 0.7651264667510986 (0.7811792029277971) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3403 (0.24443578557311144) + | > loader_time: 0.0127 (0.011616537781326024) + + + --> TIME: 2024-10-20 09:32:57 -- STEP: 21141/69059 -- GLOBAL_STEP: 90200 + | > loss_text_ce: 0.03230149298906326 (0.032257163707817504) + | > loss_mel_ce: 3.161419630050659 (3.0924104863819797) + | > loss: 0.7984302639961243 (0.7811669125912636) + | > current_lr: 5e-06 + | > step_time: 0.2315 (0.24441222211652736) + | > loader_time: 0.0117 (0.011616043198453203) + + + --> TIME: 2024-10-20 09:33:15 -- STEP: 21191/69059 -- GLOBAL_STEP: 90250 + | > loss_text_ce: 0.0301478561013937 (0.03225736612938657) + | > loss_mel_ce: 2.915675640106201 (3.092302338431514) + | > loss: 0.7364558577537537 (0.7811399262086792) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.2443965054476427) + | > loader_time: 0.0115 (0.011616282879133334) + + + --> TIME: 2024-10-20 09:33:34 -- STEP: 21241/69059 -- GLOBAL_STEP: 90300 + | > loss_text_ce: 0.032718226313591 (0.03225651840542084) + | > loss_mel_ce: 3.101714849472046 (3.0923000487964107) + | > loss: 0.7836082577705383 (0.7811391418680933) + | > current_lr: 5e-06 + | > step_time: 0.5887 (0.2444255241234926) + | > loader_time: 0.0151 (0.011617044221490043) + + + --> TIME: 2024-10-20 09:33:54 -- STEP: 21291/69059 -- GLOBAL_STEP: 90350 + | > loss_text_ce: 0.03212756663560867 (0.03225636435259288) + | > loss_mel_ce: 3.0609359741210938 (3.0923191315749903) + | > loss: 0.7732658982276917 (0.7811438740545559) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2333 (0.24448391021790158) + | > loader_time: 0.0113 (0.011617429875357757) + + + --> TIME: 2024-10-20 09:34:13 -- STEP: 21341/69059 -- GLOBAL_STEP: 90400 + | > loss_text_ce: 0.034361325204372406 (0.032256029877314504) + | > loss_mel_ce: 3.085230827331543 (3.092237884027136) + | > loss: 0.7798980474472046 (0.781123478543562) + | > current_lr: 5e-06 + | > step_time: 0.1959 (0.24449637724809067) + | > loader_time: 0.0106 (0.011617133590700383) + + + --> TIME: 2024-10-20 09:34:32 -- STEP: 21391/69059 -- GLOBAL_STEP: 90450 + | > loss_text_ce: 0.030471056699752808 (0.03225512785856137) + | > loss_mel_ce: 2.822929620742798 (3.0921949276855534) + | > loss: 0.7133501768112183 (0.781112513960809) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2036 (0.2445180881553388) + | > loader_time: 0.0105 (0.011616198568846563) + + + --> TIME: 2024-10-20 09:34:50 -- STEP: 21441/69059 -- GLOBAL_STEP: 90500 + | > loss_text_ce: 0.03193134814500809 (0.032254150156851825) + | > loss_mel_ce: 3.272268295288086 (3.0921512844346535) + | > loss: 0.8260499238967896 (0.7811013587230252) + | > current_lr: 5e-06 + | > step_time: 0.2774 (0.2445128976695491) + | > loader_time: 0.0118 (0.0116159463239013) + + + --> TIME: 2024-10-20 09:35:08 -- STEP: 21491/69059 -- GLOBAL_STEP: 90550 + | > loss_text_ce: 0.030473284423351288 (0.032253735770999895) + | > loss_mel_ce: 3.032238483428955 (3.0920529901817555) + | > loss: 0.7656779289245605 (0.7810766815689916) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2267 (0.24448355344412284) + | > loader_time: 0.0115 (0.011615632760852576) + + + --> TIME: 2024-10-20 09:35:26 -- STEP: 21541/69059 -- GLOBAL_STEP: 90600 + | > loss_text_ce: 0.02978547476232052 (0.0322528528128709) + | > loss_mel_ce: 3.158379554748535 (3.0920655078461876) + | > loss: 0.7970412373542786 (0.7810795902431099) + | > current_lr: 5e-06 + | > step_time: 0.2313 (0.2444641411589479) + | > loader_time: 0.0127 (0.011615598729148077) + + + --> TIME: 2024-10-20 09:35:44 -- STEP: 21591/69059 -- GLOBAL_STEP: 90650 + | > loss_text_ce: 0.03163382411003113 (0.03225164155544259) + | > loss_mel_ce: 2.94722318649292 (3.0920623904562814) + | > loss: 0.7447142601013184 (0.7810785080867455) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24445993236313662) + | > loader_time: 0.0102 (0.01161544018525853) + + + --> TIME: 2024-10-20 09:36:04 -- STEP: 21641/69059 -- GLOBAL_STEP: 90700 + | > loss_text_ce: 0.03198518604040146 (0.032250244830844016) + | > loss_mel_ce: 3.3563923835754395 (3.0921080491882087) + | > loss: 0.8470944166183472 (0.7810895735876741) + | > current_lr: 5e-06 + | > step_time: 0.6023 (0.24450689798516134) + | > loader_time: 0.0151 (0.01161526146373975) + + + --> TIME: 2024-10-20 09:36:24 -- STEP: 21691/69059 -- GLOBAL_STEP: 90750 + | > loss_text_ce: 0.032432671636343 (0.032249758421494135) + | > loss_mel_ce: 2.7441675662994385 (3.092111478487825) + | > loss: 0.6941500306129456 (0.7810903093109296) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2168 (0.24454465789983107) + | > loader_time: 0.0105 (0.011615275149578191) + + + --> TIME: 2024-10-20 09:36:43 -- STEP: 21741/69059 -- GLOBAL_STEP: 90800 + | > loss_text_ce: 0.031779348850250244 (0.03224988383088122) + | > loss_mel_ce: 3.035005807876587 (3.0920554959046243) + | > loss: 0.7666962742805481 (0.7810763450188903) + | > current_lr: 5e-06 + | > step_time: 0.2467 (0.24455793864220057) + | > loader_time: 0.0105 (0.01161555990355361) + + + --> TIME: 2024-10-20 09:37:01 -- STEP: 21791/69059 -- GLOBAL_STEP: 90850 + | > loss_text_ce: 0.03056989051401615 (0.03224913202202925) + | > loss_mel_ce: 3.05887508392334 (3.0920491593629325) + | > loss: 0.7723612189292908 (0.7810745729229817) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2283 (0.24456792061339072) + | > loader_time: 0.011 (0.011615092930188269) + + + --> TIME: 2024-10-20 09:37:19 -- STEP: 21841/69059 -- GLOBAL_STEP: 90900 + | > loss_text_ce: 0.0298331119120121 (0.03224865027023791) + | > loss_mel_ce: 2.7345352172851562 (3.092006460639285) + | > loss: 0.6910920739173889 (0.7810637777958871) + | > current_lr: 5e-06 + | > step_time: 0.1868 (0.2445287934033395) + | > loader_time: 0.0108 (0.01161493986966457) + + + --> TIME: 2024-10-20 09:37:38 -- STEP: 21891/69059 -- GLOBAL_STEP: 90950 + | > loss_text_ce: 0.02949359640479088 (0.03224751105566356) + | > loss_mel_ce: 3.0947022438049316 (3.091903747274362) + | > loss: 0.7810489535331726 (0.7810378146446024) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2074 (0.24453864164684824) + | > loader_time: 0.0121 (0.011615371002710278) + + + --> TIME: 2024-10-20 09:37:57 -- STEP: 21941/69059 -- GLOBAL_STEP: 91000 + | > loss_text_ce: 0.03348543867468834 (0.03224699911174809) + | > loss_mel_ce: 3.097770929336548 (3.091892565988054) + | > loss: 0.782814085483551 (0.7810348913329574) + | > current_lr: 5e-06 + | > step_time: 0.1943 (0.2445671205441089) + | > loader_time: 0.0101 (0.011615583430579226) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_91000.pth + + --> TIME: 2024-10-20 09:38:34 -- STEP: 21991/69059 -- GLOBAL_STEP: 91050 + | > loss_text_ce: 0.034395359456539154 (0.032246575392110316) + | > loss_mel_ce: 2.9811384677886963 (3.0918294177103554) + | > loss: 0.7538834810256958 (0.7810189983300612) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.24457162986981926) + | > loader_time: 0.0117 (0.011616311377519738) + + + --> TIME: 2024-10-20 09:38:53 -- STEP: 22041/69059 -- GLOBAL_STEP: 91100 + | > loss_text_ce: 0.03130154311656952 (0.03224609926228723) + | > loss_mel_ce: 2.935088634490967 (3.0917742170248292) + | > loss: 0.7415975332260132 (0.7810050791196144) + | > current_lr: 5e-06 + | > step_time: 0.1956 (0.24457929878690052) + | > loader_time: 0.0105 (0.011616658798812454) + + + --> TIME: 2024-10-20 09:39:12 -- STEP: 22091/69059 -- GLOBAL_STEP: 91150 + | > loss_text_ce: 0.0328475721180439 (0.03224560959976871) + | > loss_mel_ce: 3.165210008621216 (3.0918824664496465) + | > loss: 0.7995144128799438 (0.7810320190633059) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3356 (0.24460820704705316) + | > loader_time: 0.0127 (0.01161719177972736) + + + --> TIME: 2024-10-20 09:39:31 -- STEP: 22141/69059 -- GLOBAL_STEP: 91200 + | > loss_text_ce: 0.03275167569518089 (0.03224557135773216) + | > loss_mel_ce: 3.5669291019439697 (3.091886056201196) + | > loss: 0.8999201655387878 (0.7810329069403588) + | > current_lr: 5e-06 + | > step_time: 0.3377 (0.24462285050854374) + | > loader_time: 0.0127 (0.011617280868251853) + + + --> TIME: 2024-10-20 09:39:50 -- STEP: 22191/69059 -- GLOBAL_STEP: 91250 + | > loss_text_ce: 0.031275056302547455 (0.032245074620619193) + | > loss_mel_ce: 3.1617281436920166 (3.091876093797942) + | > loss: 0.7982507944107056 (0.7810302921598743) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3123 (0.24463322486987468) + | > loader_time: 0.0118 (0.011617562086562524) + + + --> TIME: 2024-10-20 09:40:08 -- STEP: 22241/69059 -- GLOBAL_STEP: 91300 + | > loss_text_ce: 0.030619751662015915 (0.032244071222784595) + | > loss_mel_ce: 2.819078207015991 (3.0918367689212376) + | > loss: 0.7124245166778564 (0.7810202100943606) + | > current_lr: 5e-06 + | > step_time: 0.177 (0.24463368123568505) + | > loader_time: 0.0103 (0.011616832022990457) + + + --> TIME: 2024-10-20 09:40:27 -- STEP: 22291/69059 -- GLOBAL_STEP: 91350 + | > loss_text_ce: 0.03131524845957756 (0.032243325506753635) + | > loss_mel_ce: 2.981539249420166 (3.0918688722020513) + | > loss: 0.75321364402771 (0.7810280494882248) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2472 (0.2446489665032418) + | > loader_time: 0.0107 (0.011616683873710551) + + + --> TIME: 2024-10-20 09:40:46 -- STEP: 22341/69059 -- GLOBAL_STEP: 91400 + | > loss_text_ce: 0.0352693609893322 (0.032243400810481715) + | > loss_mel_ce: 3.2883622646331787 (3.09185238976458) + | > loss: 0.8309078812599182 (0.7810239477078832) + | > current_lr: 5e-06 + | > step_time: 0.1851 (0.24464938549990947) + | > loader_time: 0.0097 (0.011615522833798079) + + + --> TIME: 2024-10-20 09:41:05 -- STEP: 22391/69059 -- GLOBAL_STEP: 91450 + | > loss_text_ce: 0.032339274883270264 (0.032242267122259013) + | > loss_mel_ce: 2.95031476020813 (3.0918167535895766) + | > loss: 0.7456635236740112 (0.7810147552384603) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4652 (0.2446658198476014) + | > loader_time: 0.0134 (0.011615138530709907) + + + --> TIME: 2024-10-20 09:41:23 -- STEP: 22441/69059 -- GLOBAL_STEP: 91500 + | > loss_text_ce: 0.03482739254832268 (0.03224194448816948) + | > loss_mel_ce: 3.4659249782562256 (3.0919054726388473) + | > loss: 0.8751881122589111 (0.7810368543480555) + | > current_lr: 5e-06 + | > step_time: 0.2071 (0.24465686355617347) + | > loader_time: 0.0146 (0.011614577654747663) + + + --> TIME: 2024-10-20 09:41:40 -- STEP: 22491/69059 -- GLOBAL_STEP: 91550 + | > loss_text_ce: 0.030833706259727478 (0.032240747845114465) + | > loss_mel_ce: 3.1136603355407715 (3.0917324893688876) + | > loss: 0.786123514175415 (0.7809933093627598) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2583 (0.24460603350091129) + | > loader_time: 0.0115 (0.011613019135724361) + + + --> TIME: 2024-10-20 09:42:00 -- STEP: 22541/69059 -- GLOBAL_STEP: 91600 + | > loss_text_ce: 0.030690619722008705 (0.032241176804007614) + | > loss_mel_ce: 3.1578903198242188 (3.091660507516146) + | > loss: 0.7971452474594116 (0.7809754211463348) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.2446338637825718) + | > loader_time: 0.0113 (0.011613784403932334) + + + --> TIME: 2024-10-20 09:42:20 -- STEP: 22591/69059 -- GLOBAL_STEP: 91650 + | > loss_text_ce: 0.030684107914566994 (0.03224095989817315) + | > loss_mel_ce: 2.9988749027252197 (3.091602712295771) + | > loss: 0.7573897242546082 (0.7809609181060608) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.284 (0.2447016991750426) + | > loader_time: 0.011 (0.011613530311856338) + + + --> TIME: 2024-10-20 09:42:39 -- STEP: 22641/69059 -- GLOBAL_STEP: 91700 + | > loss_text_ce: 0.032893236726522446 (0.03224070114613212) + | > loss_mel_ce: 3.2536725997924805 (3.091591298182632) + | > loss: 0.8216414451599121 (0.7809579998937932) + | > current_lr: 5e-06 + | > step_time: 0.1948 (0.24471923796449593) + | > loader_time: 0.011 (0.011613289915332264) + + + --> TIME: 2024-10-20 09:42:57 -- STEP: 22691/69059 -- GLOBAL_STEP: 91750 + | > loss_text_ce: 0.03158453479409218 (0.0322395587607358) + | > loss_mel_ce: 3.0231964588165283 (3.091533296961729) + | > loss: 0.763695240020752 (0.7809432139923494) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1907 (0.24471115040477565) + | > loader_time: 0.0106 (0.011612429466901528) + + + --> TIME: 2024-10-20 09:43:15 -- STEP: 22741/69059 -- GLOBAL_STEP: 91800 + | > loss_text_ce: 0.030065780505537987 (0.03223854731880604) + | > loss_mel_ce: 2.7667226791381836 (3.0914276923800172) + | > loss: 0.6991971135139465 (0.7809165599797917) + | > current_lr: 5e-06 + | > step_time: 0.1774 (0.244677018681028) + | > loader_time: 0.0107 (0.011610770811585262) + + + --> TIME: 2024-10-20 09:43:33 -- STEP: 22791/69059 -- GLOBAL_STEP: 91850 + | > loss_text_ce: 0.033700715750455856 (0.03223751747545048) + | > loss_mel_ce: 3.6307191848754883 (3.091440992155315) + | > loss: 0.9161049723625183 (0.7809196274725455) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3087 (0.24466134817602037) + | > loader_time: 0.0117 (0.01160924371163082) + + + --> TIME: 2024-10-20 09:43:51 -- STEP: 22841/69059 -- GLOBAL_STEP: 91900 + | > loss_text_ce: 0.03163135424256325 (0.032236185176662324) + | > loss_mel_ce: 3.0245463848114014 (3.091437242592761) + | > loss: 0.764044463634491 (0.7809183570007886) + | > current_lr: 5e-06 + | > step_time: 0.384 (0.24464244057786572) + | > loader_time: 0.0124 (0.011608286020003281) + + + --> TIME: 2024-10-20 09:44:09 -- STEP: 22891/69059 -- GLOBAL_STEP: 91950 + | > loss_text_ce: 0.03335660323500633 (0.032235477799206935) + | > loss_mel_ce: 2.8427445888519287 (3.0913423039873344) + | > loss: 0.7190253138542175 (0.7808944455064255) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1905 (0.24462763056322417) + | > loader_time: 0.0098 (0.011606552691370293) + + + --> TIME: 2024-10-20 09:44:28 -- STEP: 22941/69059 -- GLOBAL_STEP: 92000 + | > loss_text_ce: 0.032005682587623596 (0.032234889771101824) + | > loss_mel_ce: 3.389267921447754 (3.091352800582755) + | > loss: 0.8553184270858765 (0.7808969226410806) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.2446658961427155) + | > loader_time: 0.0098 (0.011605375070090929) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_92000.pth + + --> TIME: 2024-10-20 09:45:00 -- STEP: 22991/69059 -- GLOBAL_STEP: 92050 + | > loss_text_ce: 0.03160123899579048 (0.032235249382209834) + | > loss_mel_ce: 2.9423694610595703 (3.09130952357311) + | > loss: 0.7434926629066467 (0.780886193278106) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2086 (0.2446672756505674) + | > loader_time: 0.01 (0.011605592908681305) + + + --> TIME: 2024-10-20 09:45:18 -- STEP: 23041/69059 -- GLOBAL_STEP: 92100 + | > loss_text_ce: 0.03326014056801796 (0.03223388874804312) + | > loss_mel_ce: 3.2021071910858154 (3.0913206793917105) + | > loss: 0.8088418245315552 (0.7808886420798689) + | > current_lr: 5e-06 + | > step_time: 0.4495 (0.2446626674321309) + | > loader_time: 0.0141 (0.01160505245214204) + + + --> TIME: 2024-10-20 09:45:38 -- STEP: 23091/69059 -- GLOBAL_STEP: 92150 + | > loss_text_ce: 0.030394738540053368 (0.032233250365280935) + | > loss_mel_ce: 2.922321081161499 (3.0913531307762834) + | > loss: 0.7381789684295654 (0.7808965953182858) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2373 (0.24470084517139748) + | > loader_time: 0.0111 (0.011605672882560491) + + + --> TIME: 2024-10-20 09:45:57 -- STEP: 23141/69059 -- GLOBAL_STEP: 92200 + | > loss_text_ce: 0.032371945679187775 (0.03223183495973561) + | > loss_mel_ce: 3.5034031867980957 (3.0913516144822206) + | > loss: 0.8839437961578369 (0.7808958623991483) + | > current_lr: 5e-06 + | > step_time: 0.5246 (0.24472788864758208) + | > loader_time: 0.0145 (0.011605357944509464) + + + --> TIME: 2024-10-20 09:46:17 -- STEP: 23191/69059 -- GLOBAL_STEP: 92250 + | > loss_text_ce: 0.03396490588784218 (0.03223144804498158) + | > loss_mel_ce: 3.495620012283325 (3.091274586178678) + | > loss: 0.8823962211608887 (0.7808765085958361) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4919 (0.24475140289081954) + | > loader_time: 0.0129 (0.011604857503362898) + + + --> TIME: 2024-10-20 09:46:35 -- STEP: 23241/69059 -- GLOBAL_STEP: 92300 + | > loss_text_ce: 0.03136863559484482 (0.032230927052582306) + | > loss_mel_ce: 2.9410364627838135 (3.0913192246683416) + | > loss: 0.7431012988090515 (0.7808875379794833) + | > current_lr: 5e-06 + | > step_time: 0.1957 (0.24473928991793506) + | > loader_time: 0.0123 (0.011604910375718936) + + + --> TIME: 2024-10-20 09:46:53 -- STEP: 23291/69059 -- GLOBAL_STEP: 92350 + | > loss_text_ce: 0.03295312449336052 (0.03223034345652577) + | > loss_mel_ce: 3.4579734802246094 (3.0911955030585347) + | > loss: 0.872731626033783 (0.7808564616853692) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5406 (0.24473987636629227) + | > loader_time: 0.0135 (0.01160472803171941) + + + --> TIME: 2024-10-20 09:47:10 -- STEP: 23341/69059 -- GLOBAL_STEP: 92400 + | > loss_text_ce: 0.033909235149621964 (0.032229098994798144) + | > loss_mel_ce: 3.200737237930298 (3.0910231735236207) + | > loss: 0.8086616396903992 (0.7808130681936089) + | > current_lr: 5e-06 + | > step_time: 0.4284 (0.24469375030163262) + | > loader_time: 0.0128 (0.011603409177279148) + + + --> TIME: 2024-10-20 09:47:29 -- STEP: 23391/69059 -- GLOBAL_STEP: 92450 + | > loss_text_ce: 0.02925179898738861 (0.03222819646900632) + | > loss_mel_ce: 3.1370370388031006 (3.0908602408287105) + | > loss: 0.7915722131729126 (0.7807721093906655) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2458 (0.2446893002619868) + | > loader_time: 0.0113 (0.011602807170353719) + + + --> TIME: 2024-10-20 09:47:47 -- STEP: 23441/69059 -- GLOBAL_STEP: 92500 + | > loss_text_ce: 0.03011765144765377 (0.03222750334621325) + | > loss_mel_ce: 3.253481864929199 (3.090760529445573) + | > loss: 0.8208999037742615 (0.7807470082646772) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.24469331849593215) + | > loader_time: 0.011 (0.01160195364967751) + + + --> TIME: 2024-10-20 09:48:05 -- STEP: 23491/69059 -- GLOBAL_STEP: 92550 + | > loss_text_ce: 0.03239426016807556 (0.03222682370523692) + | > loss_mel_ce: 2.814033269882202 (3.0906254977975984) + | > loss: 0.7116068601608276 (0.780713080441346) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1954 (0.24469147868897548) + | > loader_time: 0.0105 (0.011601833288130962) + + + --> TIME: 2024-10-20 09:48:26 -- STEP: 23541/69059 -- GLOBAL_STEP: 92600 + | > loss_text_ce: 0.03359353169798851 (0.032226435330337644) + | > loss_mel_ce: 3.1694207191467285 (3.0906895567873294) + | > loss: 0.8007535338401794 (0.7807289981015609) + | > current_lr: 5e-06 + | > step_time: 0.3218 (0.24476454092983208) + | > loader_time: 0.0121 (0.011601479880634879) + + + --> TIME: 2024-10-20 09:48:45 -- STEP: 23591/69059 -- GLOBAL_STEP: 92650 + | > loss_text_ce: 0.03209026902914047 (0.03222597621935255) + | > loss_mel_ce: 3.446082592010498 (3.090640709867176) + | > loss: 0.869543194770813 (0.7807166716004531) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4207 (0.24475895076501242) + | > loader_time: 0.0123 (0.011600190628722497) + + + --> TIME: 2024-10-20 09:49:02 -- STEP: 23641/69059 -- GLOBAL_STEP: 92700 + | > loss_text_ce: 0.030920542776584625 (0.03222445880283622) + | > loss_mel_ce: 3.228973388671875 (3.0905937714542486) + | > loss: 0.8149734735488892 (0.7807045576384936) + | > current_lr: 5e-06 + | > step_time: 0.205 (0.244726055099357) + | > loader_time: 0.0108 (0.011599484083389382) + + + --> TIME: 2024-10-20 09:49:20 -- STEP: 23691/69059 -- GLOBAL_STEP: 92750 + | > loss_text_ce: 0.029912587255239487 (0.03222370277974678) + | > loss_mel_ce: 2.8338093757629395 (3.0904764360828083) + | > loss: 0.7159304618835449 (0.7806750347899993) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2048 (0.24471794575204994) + | > loader_time: 0.0113 (0.011598758843231815) + + + --> TIME: 2024-10-20 09:49:39 -- STEP: 23741/69059 -- GLOBAL_STEP: 92800 + | > loss_text_ce: 0.03125159442424774 (0.03222319667055882) + | > loss_mel_ce: 2.948371648788452 (3.0904241034016446) + | > loss: 0.7449058294296265 (0.7806618250860959) + | > current_lr: 5e-06 + | > step_time: 0.187 (0.24474554219355782) + | > loader_time: 0.0106 (0.011598346478504922) + + + --> TIME: 2024-10-20 09:49:57 -- STEP: 23791/69059 -- GLOBAL_STEP: 92850 + | > loss_text_ce: 0.03383161872625351 (0.03222164863477928) + | > loss_mel_ce: 2.890608787536621 (3.090363951289751) + | > loss: 0.7311100959777832 (0.7806464000496022) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2463 (0.2447174761415676) + | > loader_time: 0.0106 (0.011597622789263722) + + + --> TIME: 2024-10-20 09:50:15 -- STEP: 23841/69059 -- GLOBAL_STEP: 92900 + | > loss_text_ce: 0.031901221722364426 (0.03221985482824501) + | > loss_mel_ce: 2.8937132358551025 (3.0903114751703087) + | > loss: 0.7314035892486572 (0.7806328325622416) + | > current_lr: 5e-06 + | > step_time: 0.2016 (0.24468575759570252) + | > loader_time: 0.0112 (0.011596196370108619) + + + --> TIME: 2024-10-20 09:50:33 -- STEP: 23891/69059 -- GLOBAL_STEP: 92950 + | > loss_text_ce: 0.03332674503326416 (0.03221992465624686) + | > loss_mel_ce: 3.106419563293457 (3.0901838790267018) + | > loss: 0.7849365472793579 (0.7806009509844763) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2581 (0.24466933985951214) + | > loader_time: 0.0102 (0.011595309761149235) + + + --> TIME: 2024-10-20 09:50:52 -- STEP: 23941/69059 -- GLOBAL_STEP: 93000 + | > loss_text_ce: 0.031761039048433304 (0.03221981379663214) + | > loss_mel_ce: 3.4237892627716064 (3.0902367960186137) + | > loss: 0.8638875484466553 (0.7806141525177092) + | > current_lr: 5e-06 + | > step_time: 0.2349 (0.24470939416544002) + | > loader_time: 0.011 (0.011594545312913897) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_93000.pth + + --> TIME: 2024-10-20 09:51:25 -- STEP: 23991/69059 -- GLOBAL_STEP: 93050 + | > loss_text_ce: 0.03157610073685646 (0.03221872102642056) + | > loss_mel_ce: 3.2296197414398193 (3.0902715307094613) + | > loss: 0.8152989745140076 (0.780622563002471) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.216 (0.244683043992831) + | > loader_time: 0.0107 (0.011594402214172093) + + + --> TIME: 2024-10-20 09:51:44 -- STEP: 24041/69059 -- GLOBAL_STEP: 93100 + | > loss_text_ce: 0.031820476055145264 (0.032216965653156515) + | > loss_mel_ce: 2.9094812870025635 (3.090246863781878) + | > loss: 0.7353254556655884 (0.7806159574221582) + | > current_lr: 5e-06 + | > step_time: 0.1717 (0.24468917366887222) + | > loader_time: 0.0109 (0.011594063588790544) + + + --> TIME: 2024-10-20 09:52:02 -- STEP: 24091/69059 -- GLOBAL_STEP: 93150 + | > loss_text_ce: 0.032870300114154816 (0.03221590186647377) + | > loss_mel_ce: 3.2133519649505615 (3.090207799355534) + | > loss: 0.8115555644035339 (0.7806059253710121) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2925 (0.2446849710355141) + | > loader_time: 0.0117 (0.011593926329476359) + + + --> TIME: 2024-10-20 09:52:19 -- STEP: 24141/69059 -- GLOBAL_STEP: 93200 + | > loss_text_ce: 0.034376926720142365 (0.03221532155580918) + | > loss_mel_ce: 2.928571939468384 (3.09006648998726) + | > loss: 0.7407371997833252 (0.7805704529434261) + | > current_lr: 5e-06 + | > step_time: 0.1743 (0.24461455101412027) + | > loader_time: 0.014 (0.011593053830864467) + + + --> TIME: 2024-10-20 09:52:37 -- STEP: 24191/69059 -- GLOBAL_STEP: 93250 + | > loss_text_ce: 0.03094324842095375 (0.03221510446804719) + | > loss_mel_ce: 3.136284112930298 (3.0900809684833854) + | > loss: 0.7918068170547485 (0.7805740182881985) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1958 (0.24461820809464974) + | > loader_time: 0.0104 (0.011592716238565807) + + + --> TIME: 2024-10-20 09:52:55 -- STEP: 24241/69059 -- GLOBAL_STEP: 93300 + | > loss_text_ce: 0.03312113508582115 (0.032214661032425096) + | > loss_mel_ce: 3.21256685256958 (3.0900788686932437) + | > loss: 0.8114219903945923 (0.7805733824862064) + | > current_lr: 5e-06 + | > step_time: 0.5374 (0.24461583831336736) + | > loader_time: 0.015 (0.011592353906407619) + + + --> TIME: 2024-10-20 09:53:14 -- STEP: 24291/69059 -- GLOBAL_STEP: 93350 + | > loss_text_ce: 0.033411215990781784 (0.032214359422901) + | > loss_mel_ce: 3.1249001026153564 (3.0900043092039438) + | > loss: 0.789577841758728 (0.7805546672139756) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2488 (0.24462483788149092) + | > loader_time: 0.0113 (0.011591900440787855) + + + --> TIME: 2024-10-20 09:53:33 -- STEP: 24341/69059 -- GLOBAL_STEP: 93400 + | > loss_text_ce: 0.03457958251237869 (0.032213688363824794) + | > loss_mel_ce: 3.1837832927703857 (3.0900588865606373) + | > loss: 0.8045907020568848 (0.7805681437857562) + | > current_lr: 5e-06 + | > step_time: 0.2348 (0.24463917567409565) + | > loader_time: 0.0104 (0.011591801465691415) + + + --> TIME: 2024-10-20 09:53:53 -- STEP: 24391/69059 -- GLOBAL_STEP: 93450 + | > loss_text_ce: 0.03263827785849571 (0.032212435785284266) + | > loss_mel_ce: 3.046097755432129 (3.0900718750761356) + | > loss: 0.7696840167045593 (0.7805710777669054) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4104 (0.24468281973743058) + | > loader_time: 0.0146 (0.011592020501089902) + + + --> TIME: 2024-10-20 09:54:12 -- STEP: 24441/69059 -- GLOBAL_STEP: 93500 + | > loss_text_ce: 0.034642793238162994 (0.03221216810205174) + | > loss_mel_ce: 3.2109100818634033 (3.090084251603028) + | > loss: 0.8113881945610046 (0.7805741049694078) + | > current_lr: 5e-06 + | > step_time: 0.259 (0.24468303146204673) + | > loader_time: 0.0118 (0.011592136672737022) + + + --> TIME: 2024-10-20 09:54:29 -- STEP: 24491/69059 -- GLOBAL_STEP: 93550 + | > loss_text_ce: 0.03443098068237305 (0.0322116509156446) + | > loss_mel_ce: 2.5687663555145264 (3.08999986164926) + | > loss: 0.6507993340492249 (0.7805528781806444) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1952 (0.24465130530119797) + | > loader_time: 0.0103 (0.01159177796661837) + + + --> TIME: 2024-10-20 09:54:47 -- STEP: 24541/69059 -- GLOBAL_STEP: 93600 + | > loss_text_ce: 0.031745750457048416 (0.03221155800499758) + | > loss_mel_ce: 2.7689688205718994 (3.089977713631091) + | > loss: 0.7001786231994629 (0.7805473179452103) + | > current_lr: 5e-06 + | > step_time: 0.1943 (0.24464177351836153) + | > loader_time: 0.0108 (0.011591389721235872) + + + --> TIME: 2024-10-20 09:55:06 -- STEP: 24591/69059 -- GLOBAL_STEP: 93650 + | > loss_text_ce: 0.03159895911812782 (0.03221041676515854) + | > loss_mel_ce: 3.219876527786255 (3.0899617731786777) + | > loss: 0.8128688931465149 (0.7805430475307464) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.287 (0.24465520162521517) + | > loader_time: 0.0127 (0.01159137004340225) + + + --> TIME: 2024-10-20 09:55:24 -- STEP: 24641/69059 -- GLOBAL_STEP: 93700 + | > loss_text_ce: 0.033802904188632965 (0.03220972776256362) + | > loss_mel_ce: 2.9947595596313477 (3.0899248417212903) + | > loss: 0.7571406364440918 (0.7805336424173419) + | > current_lr: 5e-06 + | > step_time: 0.1867 (0.24462857149061532) + | > loader_time: 0.011 (0.011590799753757168) + + + --> TIME: 2024-10-20 09:55:43 -- STEP: 24691/69059 -- GLOBAL_STEP: 93750 + | > loss_text_ce: 0.0316237173974514 (0.0322087871524154) + | > loss_mel_ce: 2.997878074645996 (3.08986168439571) + | > loss: 0.7573754191398621 (0.780517617941897) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2062 (0.2446406315175535) + | > loader_time: 0.0104 (0.011590540730153212) + + + --> TIME: 2024-10-20 09:56:02 -- STEP: 24741/69059 -- GLOBAL_STEP: 93800 + | > loss_text_ce: 0.02987160161137581 (0.032208339414060726) + | > loss_mel_ce: 2.8997085094451904 (3.0898614016284864) + | > loss: 0.7323950529098511 (0.7805174353229578) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.2446638087120811) + | > loader_time: 0.0109 (0.011590426020496787) + + + --> TIME: 2024-10-20 09:56:19 -- STEP: 24791/69059 -- GLOBAL_STEP: 93850 + | > loss_text_ce: 0.031729213893413544 (0.0322077581530107) + | > loss_mel_ce: 2.6624414920806885 (3.0897615177096234) + | > loss: 0.6735426783561707 (0.7804923190248673) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2153 (0.24463526162319907) + | > loader_time: 0.0118 (0.011589854468805382) + + + --> TIME: 2024-10-20 09:56:38 -- STEP: 24841/69059 -- GLOBAL_STEP: 93900 + | > loss_text_ce: 0.03158154711127281 (0.032206692958681185) + | > loss_mel_ce: 3.0437049865722656 (3.089745208110529) + | > loss: 0.768821656703949 (0.780487975330966) + | > current_lr: 5e-06 + | > step_time: 0.1953 (0.2446454400280278) + | > loader_time: 0.0105 (0.011589831015198069) + + + --> TIME: 2024-10-20 09:56:58 -- STEP: 24891/69059 -- GLOBAL_STEP: 93950 + | > loss_text_ce: 0.03310984745621681 (0.03220673999302961) + | > loss_mel_ce: 2.753044605255127 (3.0896638607251674) + | > loss: 0.6965386271476746 (0.7804676502443756) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1981 (0.24467283605375034) + | > loader_time: 0.0103 (0.011589655941868218) + + + --> TIME: 2024-10-20 09:57:16 -- STEP: 24941/69059 -- GLOBAL_STEP: 94000 + | > loss_text_ce: 0.031233809888362885 (0.03220562895670421) + | > loss_mel_ce: 2.843017578125 (3.0895968634173685) + | > loss: 0.7185628414154053 (0.7804506231695476) + | > current_lr: 5e-06 + | > step_time: 0.1784 (0.2446531923478448) + | > loader_time: 0.0101 (0.011588643659332659) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_94000.pth + + --> TIME: 2024-10-20 09:57:48 -- STEP: 24991/69059 -- GLOBAL_STEP: 94050 + | > loss_text_ce: 0.029856383800506592 (0.032204318137827856) + | > loss_mel_ce: 3.0634937286376953 (3.089580739619151) + | > loss: 0.7733375430107117 (0.7804462645101657) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2268 (0.244648276951022) + | > loader_time: 0.0109 (0.01158851493636098) + + + --> TIME: 2024-10-20 09:58:08 -- STEP: 25041/69059 -- GLOBAL_STEP: 94100 + | > loss_text_ce: 0.02904791198670864 (0.03220324703508576) + | > loss_mel_ce: 2.8932607173919678 (3.0895546968502674) + | > loss: 0.7305771708488464 (0.780439486043159) + | > current_lr: 5e-06 + | > step_time: 0.1788 (0.24467641027356196) + | > loader_time: 0.0111 (0.011589028756176055) + + + --> TIME: 2024-10-20 09:58:25 -- STEP: 25091/69059 -- GLOBAL_STEP: 94150 + | > loss_text_ce: 0.03493613004684448 (0.03220151468748011) + | > loss_mel_ce: 3.165210008621216 (3.0895814610184997) + | > loss: 0.8000365495681763 (0.7804457439925306) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3796 (0.24464464447981132) + | > loader_time: 0.0128 (0.011588365748197438) + + + --> TIME: 2024-10-20 09:58:44 -- STEP: 25141/69059 -- GLOBAL_STEP: 94200 + | > loss_text_ce: 0.028646614402532578 (0.03220040548907999) + | > loss_mel_ce: 2.8924286365509033 (3.089427363326753) + | > loss: 0.7302688360214233 (0.7804069422645652) + | > current_lr: 5e-06 + | > step_time: 0.1771 (0.24465150591496487) + | > loader_time: 0.0117 (0.011588305601199885) + + + --> TIME: 2024-10-20 09:59:01 -- STEP: 25191/69059 -- GLOBAL_STEP: 94250 + | > loss_text_ce: 0.030967649072408676 (0.03219884005446401) + | > loss_mel_ce: 3.2808990478515625 (3.089360499087661) + | > loss: 0.8279666900634766 (0.7803898348562589) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2005 (0.2446114222587237) + | > loader_time: 0.0132 (0.011587594312699876) + + + --> TIME: 2024-10-20 09:59:19 -- STEP: 25241/69059 -- GLOBAL_STEP: 94300 + | > loss_text_ce: 0.030840273946523666 (0.03219882591218482) + | > loss_mel_ce: 2.9996418952941895 (3.0892774009613775) + | > loss: 0.7576205134391785 (0.7803690567843474) + | > current_lr: 5e-06 + | > step_time: 0.1847 (0.24457208535301064) + | > loader_time: 0.011 (0.011587009429553815) + + + --> TIME: 2024-10-20 09:59:38 -- STEP: 25291/69059 -- GLOBAL_STEP: 94350 + | > loss_text_ce: 0.031206799671053886 (0.03219847043944614) + | > loss_mel_ce: 2.7938902378082275 (3.0891653133934587) + | > loss: 0.7062742710113525 (0.7803409460303496) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2163 (0.24459531474927024) + | > loader_time: 0.0106 (0.011586968044968652) + + + --> TIME: 2024-10-20 09:59:56 -- STEP: 25341/69059 -- GLOBAL_STEP: 94400 + | > loss_text_ce: 0.030077874660491943 (0.03219783720187547) + | > loss_mel_ce: 3.1444482803344727 (3.089118778900808) + | > loss: 0.7936315536499023 (0.7803291541065825) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.24458681234484164) + | > loader_time: 0.0102 (0.011586879922758364) + + + --> TIME: 2024-10-20 10:00:14 -- STEP: 25391/69059 -- GLOBAL_STEP: 94450 + | > loss_text_ce: 0.030108366161584854 (0.032197474079298995) + | > loss_mel_ce: 3.423306941986084 (3.089078481845903) + | > loss: 0.8633538484573364 (0.7803189890662894) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2143 (0.2445595278707441) + | > loader_time: 0.0108 (0.01158642371667713) + + + --> TIME: 2024-10-20 10:00:31 -- STEP: 25441/69059 -- GLOBAL_STEP: 94500 + | > loss_text_ce: 0.031064128503203392 (0.032196022371136925) + | > loss_mel_ce: 3.1098382472991943 (3.088916495490991) + | > loss: 0.7852255702018738 (0.7802781295412753) + | > current_lr: 5e-06 + | > step_time: 0.2117 (0.2445288069391302) + | > loader_time: 0.011 (0.01158629022844698) + + + --> TIME: 2024-10-20 10:00:50 -- STEP: 25491/69059 -- GLOBAL_STEP: 94550 + | > loss_text_ce: 0.03219817951321602 (0.03219597156006407) + | > loss_mel_ce: 2.9675073623657227 (3.088831475406648) + | > loss: 0.7499263882637024 (0.7802568618226434) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2019 (0.24452275278840355) + | > loader_time: 0.0101 (0.011585964469480178) + + + --> TIME: 2024-10-20 10:01:09 -- STEP: 25541/69059 -- GLOBAL_STEP: 94600 + | > loss_text_ce: 0.034298039972782135 (0.032195752393385235) + | > loss_mel_ce: 2.5660219192504883 (3.088929911441281) + | > loss: 0.6500799655914307 (0.7802814160254531) + | > current_lr: 5e-06 + | > step_time: 0.1786 (0.24456181987693607) + | > loader_time: 0.0116 (0.011586079064086137) + + + --> TIME: 2024-10-20 10:01:27 -- STEP: 25591/69059 -- GLOBAL_STEP: 94650 + | > loss_text_ce: 0.03335779905319214 (0.032194809763380676) + | > loss_mel_ce: 3.138444423675537 (3.0889000660099155) + | > loss: 0.7929505705833435 (0.780273719002738) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2378 (0.24453750239331232) + | > loader_time: 0.0104 (0.011585428568389967) + + + --> TIME: 2024-10-20 10:01:44 -- STEP: 25641/69059 -- GLOBAL_STEP: 94700 + | > loss_text_ce: 0.029137808829545975 (0.03219324073157762) + | > loss_mel_ce: 3.0451908111572266 (3.088841693123582) + | > loss: 0.7685821652412415 (0.780258733526248) + | > current_lr: 5e-06 + | > step_time: 0.1761 (0.24450248825973625) + | > loader_time: 0.01 (0.01158463684808938) + + + --> TIME: 2024-10-20 10:02:03 -- STEP: 25691/69059 -- GLOBAL_STEP: 94750 + | > loss_text_ce: 0.0323542021214962 (0.032193800545615714) + | > loss_mel_ce: 3.1638827323913574 (3.088812053795184) + | > loss: 0.7990592122077942 (0.7802514636401957) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3437 (0.24451965247432614) + | > loader_time: 0.0122 (0.011584127312631599) + + + --> TIME: 2024-10-20 10:02:23 -- STEP: 25741/69059 -- GLOBAL_STEP: 94800 + | > loss_text_ce: 0.03514696657657623 (0.032193611933855046) + | > loss_mel_ce: 3.0242104530334473 (3.0887546899884386) + | > loss: 0.7648393511772156 (0.7802370755406904) + | > current_lr: 5e-06 + | > step_time: 0.2159 (0.2445443515700527) + | > loader_time: 0.0114 (0.011583953427803472) + + + --> TIME: 2024-10-20 10:02:41 -- STEP: 25791/69059 -- GLOBAL_STEP: 94850 + | > loss_text_ce: 0.032255690544843674 (0.03219286375636664) + | > loss_mel_ce: 3.838278293609619 (3.0887346666312334) + | > loss: 0.9676334857940674 (0.7802318826581823) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.5742 (0.24454371323947338) + | > loader_time: 0.0146 (0.01158293654950009) + + + --> TIME: 2024-10-20 10:02:59 -- STEP: 25841/69059 -- GLOBAL_STEP: 94900 + | > loss_text_ce: 0.03172270581126213 (0.03219266142591639) + | > loss_mel_ce: 3.0005922317504883 (3.088619046174139) + | > loss: 0.7580787539482117 (0.7802029269720799) + | > current_lr: 5e-06 + | > step_time: 0.1765 (0.24452748795392687) + | > loader_time: 0.0112 (0.01158241414464686) + + + --> TIME: 2024-10-20 10:03:18 -- STEP: 25891/69059 -- GLOBAL_STEP: 94950 + | > loss_text_ce: 0.031750403344631195 (0.03219188326758031) + | > loss_mel_ce: 2.997426986694336 (3.088612831526116) + | > loss: 0.7572943568229675 (0.780201178771412) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2451 (0.24453062362150807) + | > loader_time: 0.012 (0.011582725123851558) + + + --> TIME: 2024-10-20 10:03:36 -- STEP: 25941/69059 -- GLOBAL_STEP: 95000 + | > loss_text_ce: 0.0352322980761528 (0.03219081607218697) + | > loss_mel_ce: 3.2389750480651855 (3.088547270500464) + | > loss: 0.8185518383979797 (0.7801845217145381) + | > current_lr: 5e-06 + | > step_time: 0.2586 (0.24453391824221363) + | > loader_time: 0.0118 (0.01158208704772293) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_95000.pth + + --> TIME: 2024-10-20 10:04:12 -- STEP: 25991/69059 -- GLOBAL_STEP: 95050 + | > loss_text_ce: 0.034252602607011795 (0.03218992192338747) + | > loss_mel_ce: 3.3149850368499756 (3.0885168888171455) + | > loss: 0.8373094201087952 (0.7801767027482195) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.238 (0.24454015444546084) + | > loader_time: 0.0108 (0.011581810049735306) + + + --> TIME: 2024-10-20 10:04:30 -- STEP: 26041/69059 -- GLOBAL_STEP: 95100 + | > loss_text_ce: 0.0323747955262661 (0.03218951481121424) + | > loss_mel_ce: 2.8726885318756104 (3.088539444463158) + | > loss: 0.7262658476829529 (0.7801822398803064) + | > current_lr: 5e-06 + | > step_time: 0.1938 (0.24453607289812446) + | > loader_time: 0.0103 (0.01158157468350233) + + + --> TIME: 2024-10-20 10:04:49 -- STEP: 26091/69059 -- GLOBAL_STEP: 95150 + | > loss_text_ce: 0.03413630649447441 (0.03218871139065321) + | > loss_mel_ce: 3.0813162326812744 (3.0885753518920587) + | > loss: 0.7788631319999695 (0.7801910158883412) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1903 (0.24454757220665319) + | > loader_time: 0.0103 (0.011582138091422852) + + + --> TIME: 2024-10-20 10:05:08 -- STEP: 26141/69059 -- GLOBAL_STEP: 95200 + | > loss_text_ce: 0.033437058329582214 (0.03218853667060358) + | > loss_mel_ce: 2.9470696449279785 (3.0885177006941533) + | > loss: 0.7451266646385193 (0.7801765594015085) + | > current_lr: 5e-06 + | > step_time: 0.2955 (0.2445527930453242) + | > loader_time: 0.0121 (0.011581906992887232) + + + --> TIME: 2024-10-20 10:05:27 -- STEP: 26191/69059 -- GLOBAL_STEP: 95250 + | > loss_text_ce: 0.033774375915527344 (0.03218740772729459) + | > loss_mel_ce: 3.136258363723755 (3.0884318900205607) + | > loss: 0.7925081849098206 (0.7801548245004039) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.6572 (0.2445671727060066) + | > loader_time: 0.0153 (0.01158211855793513) + + + --> TIME: 2024-10-20 10:05:46 -- STEP: 26241/69059 -- GLOBAL_STEP: 95300 + | > loss_text_ce: 0.03257409855723381 (0.03218622950638348) + | > loss_mel_ce: 2.9570510387420654 (3.088386780077849) + | > loss: 0.7474063038825989 (0.7801432524629264) + | > current_lr: 5e-06 + | > step_time: 0.1949 (0.24458399616151477) + | > loader_time: 0.011 (0.011582073798825988) + + + --> TIME: 2024-10-20 10:06:04 -- STEP: 26291/69059 -- GLOBAL_STEP: 95350 + | > loss_text_ce: 0.03174285963177681 (0.03218497628630406) + | > loss_mel_ce: 3.2129809856414795 (3.088344766524963) + | > loss: 0.8111809492111206 (0.7801324357720197) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.284 (0.24458866698677262) + | > loader_time: 0.0114 (0.011582075259543235) + + + --> TIME: 2024-10-20 10:06:25 -- STEP: 26341/69059 -- GLOBAL_STEP: 95400 + | > loss_text_ce: 0.03169572353363037 (0.032185205988291055) + | > loss_mel_ce: 2.9314651489257812 (3.088263086901349) + | > loss: 0.7407902479171753 (0.7801120732898731) + | > current_lr: 5e-06 + | > step_time: 0.3265 (0.24465274654204658) + | > loader_time: 0.0121 (0.011582611144881086) + + + --> TIME: 2024-10-20 10:06:43 -- STEP: 26391/69059 -- GLOBAL_STEP: 95450 + | > loss_text_ce: 0.03257129341363907 (0.032184029412103236) + | > loss_mel_ce: 3.0087735652923584 (3.0881446046938583) + | > loss: 0.7603362202644348 (0.7800821585949902) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2232 (0.2446460459178217) + | > loader_time: 0.0108 (0.0115818239903469) + + + --> TIME: 2024-10-20 10:07:02 -- STEP: 26441/69059 -- GLOBAL_STEP: 95500 + | > loss_text_ce: 0.03066493198275566 (0.03218304938009234) + | > loss_mel_ce: 2.937004327774048 (3.0881294215628414) + | > loss: 0.7419173121452332 (0.7800781177949634) + | > current_lr: 5e-06 + | > step_time: 0.1865 (0.24464575560234228) + | > loader_time: 0.0108 (0.011581141326256228) + + + --> TIME: 2024-10-20 10:07:20 -- STEP: 26491/69059 -- GLOBAL_STEP: 95550 + | > loss_text_ce: 0.032631319016218185 (0.03218257534245079) + | > loss_mel_ce: 2.8968307971954346 (3.0880107078009957) + | > loss: 0.7323655486106873 (0.7800483208411653) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3074 (0.244643750724794) + | > loader_time: 0.0118 (0.011580823461492203) + + + --> TIME: 2024-10-20 10:07:37 -- STEP: 26541/69059 -- GLOBAL_STEP: 95600 + | > loss_text_ce: 0.03178515285253525 (0.03218253309331812) + | > loss_mel_ce: 2.823276996612549 (3.0879141769935092) + | > loss: 0.713765561580658 (0.7800241775813803) + | > current_lr: 5e-06 + | > step_time: 0.1723 (0.24461020581989332) + | > loader_time: 0.0104 (0.011579894690752383) + + + --> TIME: 2024-10-20 10:07:56 -- STEP: 26591/69059 -- GLOBAL_STEP: 95650 + | > loss_text_ce: 0.03404286131262779 (0.032182274945059285) + | > loss_mel_ce: 3.327117919921875 (3.0879154364291765) + | > loss: 0.8402901887893677 (0.7800244279011063) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3064 (0.24463126467607624) + | > loader_time: 0.012 (0.011580060278009641) + + + --> TIME: 2024-10-20 10:08:15 -- STEP: 26641/69059 -- GLOBAL_STEP: 95700 + | > loss_text_ce: 0.0295385904610157 (0.032181711563594576) + | > loss_mel_ce: 2.7624475955963135 (3.0879118696337713) + | > loss: 0.6979965567588806 (0.7800233953685792) + | > current_lr: 5e-06 + | > step_time: 0.1763 (0.24464004721970345) + | > loader_time: 0.0104 (0.01157921237866087) + + + --> TIME: 2024-10-20 10:08:34 -- STEP: 26691/69059 -- GLOBAL_STEP: 95750 + | > loss_text_ce: 0.03063066489994526 (0.03218171148384066) + | > loss_mel_ce: 2.750948190689087 (3.087824746724612) + | > loss: 0.6953946948051453 (0.7800016146119573) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.282 (0.24464763150103142) + | > loader_time: 0.0108 (0.011578539643272118) + + + --> TIME: 2024-10-20 10:08:53 -- STEP: 26741/69059 -- GLOBAL_STEP: 95800 + | > loss_text_ce: 0.031682077795267105 (0.03218166887908383) + | > loss_mel_ce: 3.0051279067993164 (3.08777048966632) + | > loss: 0.7592024803161621 (0.7799880396927048) + | > current_lr: 5e-06 + | > step_time: 0.2627 (0.24465791540109147) + | > loader_time: 0.0123 (0.011578107155687564) + + + --> TIME: 2024-10-20 10:09:11 -- STEP: 26791/69059 -- GLOBAL_STEP: 95850 + | > loss_text_ce: 0.03391251340508461 (0.03218100304646467) + | > loss_mel_ce: 3.3602850437164307 (3.0877598485337465) + | > loss: 0.8485493659973145 (0.7799852129495634) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3124 (0.24465752563677334) + | > loader_time: 0.0116 (0.011577137248708574) + + + --> TIME: 2024-10-20 10:09:30 -- STEP: 26841/69059 -- GLOBAL_STEP: 95900 + | > loss_text_ce: 0.03434532508254051 (0.032180285353852915) + | > loss_mel_ce: 2.9759843349456787 (3.0876467899373403) + | > loss: 0.7525824308395386 (0.7799567688768777) + | > current_lr: 5e-06 + | > step_time: 0.2957 (0.24465822567727827) + | > loader_time: 0.0116 (0.011576398839041338) + + + --> TIME: 2024-10-20 10:09:48 -- STEP: 26891/69059 -- GLOBAL_STEP: 95950 + | > loss_text_ce: 0.03183658421039581 (0.03217998357650191) + | > loss_mel_ce: 3.264542579650879 (3.0876085329975957) + | > loss: 0.8240947723388672 (0.7799471291935032) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2274 (0.24464962716606378) + | > loader_time: 0.0109 (0.011575397963344948) + + + --> TIME: 2024-10-20 10:10:07 -- STEP: 26941/69059 -- GLOBAL_STEP: 96000 + | > loss_text_ce: 0.029937533661723137 (0.03217886300710992) + | > loss_mel_ce: 3.2115089893341064 (3.087550491768256) + | > loss: 0.8103616237640381 (0.7799323387486882) + | > current_lr: 5e-06 + | > step_time: 0.2352 (0.24465479040494556) + | > loader_time: 0.0117 (0.011574885401020099) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_96000.pth + + --> TIME: 2024-10-20 10:10:42 -- STEP: 26991/69059 -- GLOBAL_STEP: 96050 + | > loss_text_ce: 0.032616451382637024 (0.032178321911317295) + | > loss_mel_ce: 3.143404483795166 (3.0875287993708223) + | > loss: 0.7940052151679993 (0.7799267803817496) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3241 (0.24468348198612586) + | > loader_time: 0.0117 (0.011574674715290367) + + + --> TIME: 2024-10-20 10:11:01 -- STEP: 27041/69059 -- GLOBAL_STEP: 96100 + | > loss_text_ce: 0.033882200717926025 (0.03217827691711215) + | > loss_mel_ce: 2.9229514598846436 (3.0874113432776626) + | > loss: 0.7392084002494812 (0.7798974051074704) + | > current_lr: 5e-06 + | > step_time: 0.3572 (0.24470020406481868) + | > loader_time: 0.0122 (0.011574299482457947) + + + --> TIME: 2024-10-20 10:11:19 -- STEP: 27091/69059 -- GLOBAL_STEP: 96150 + | > loss_text_ce: 0.032247394323349 (0.0321772728149147) + | > loss_mel_ce: 3.195303440093994 (3.087364922972284) + | > loss: 0.806887686252594 (0.7798855490081837) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2279 (0.2446751029365312) + | > loader_time: 0.0109 (0.011573698102541222) + + + --> TIME: 2024-10-20 10:11:37 -- STEP: 27141/69059 -- GLOBAL_STEP: 96200 + | > loss_text_ce: 0.030869383364915848 (0.032176413818980204) + | > loss_mel_ce: 2.937479257583618 (3.0872464939056306) + | > loss: 0.742087185382843 (0.7798557269942595) + | > current_lr: 5e-06 + | > step_time: 0.1941 (0.24465305638916793) + | > loader_time: 0.0108 (0.011573110946720491) + + + --> TIME: 2024-10-20 10:11:55 -- STEP: 27191/69059 -- GLOBAL_STEP: 96250 + | > loss_text_ce: 0.034130532294511795 (0.032174804963524636) + | > loss_mel_ce: 2.869831085205078 (3.0872042678987763) + | > loss: 0.7259904146194458 (0.7798447682732913) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.198 (0.24465615571976626) + | > loader_time: 0.0106 (0.011572873262250585) + + + --> TIME: 2024-10-20 10:12:13 -- STEP: 27241/69059 -- GLOBAL_STEP: 96300 + | > loss_text_ce: 0.036185745149850845 (0.032173955314790434) + | > loss_mel_ce: 2.9344983100891113 (3.0871562626230347) + | > loss: 0.742671012878418 (0.7798325545391946) + | > current_lr: 5e-06 + | > step_time: 0.2971 (0.24463985562390173) + | > loader_time: 0.0111 (0.011572252867798692) + + + --> TIME: 2024-10-20 10:12:32 -- STEP: 27291/69059 -- GLOBAL_STEP: 96350 + | > loss_text_ce: 0.032220520079135895 (0.03217289238722619) + | > loss_mel_ce: 3.1143949031829834 (3.087050648914266) + | > loss: 0.7866538763046265 (0.7798058853786801) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2154 (0.24464078638981784) + | > loader_time: 0.0103 (0.011572542914255366) + + + --> TIME: 2024-10-20 10:12:49 -- STEP: 27341/69059 -- GLOBAL_STEP: 96400 + | > loss_text_ce: 0.03425036370754242 (0.03217264909829564) + | > loss_mel_ce: 3.1391873359680176 (3.087020297784877) + | > loss: 0.7933593988418579 (0.7797982367739007) + | > current_lr: 5e-06 + | > step_time: 0.3522 (0.24462407189277785) + | > loader_time: 0.014 (0.011572139717736324) + + + --> TIME: 2024-10-20 10:13:07 -- STEP: 27391/69059 -- GLOBAL_STEP: 96450 + | > loss_text_ce: 0.030050985515117645 (0.032171235182176496) + | > loss_mel_ce: 2.9634573459625244 (3.0869386216069117) + | > loss: 0.7483770847320557 (0.7797774642456755) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2163 (0.24459927370434142) + | > loader_time: 0.0107 (0.011571742371467263) + + + --> TIME: 2024-10-20 10:13:27 -- STEP: 27441/69059 -- GLOBAL_STEP: 96500 + | > loss_text_ce: 0.031295113265514374 (0.032170627063959456) + | > loss_mel_ce: 3.1223134994506836 (3.0868140977106813) + | > loss: 0.7884021401405334 (0.7797461812439099) + | > current_lr: 5e-06 + | > step_time: 0.1943 (0.24463248004380217) + | > loader_time: 0.0118 (0.011576240852750483) + + + --> TIME: 2024-10-20 10:13:46 -- STEP: 27491/69059 -- GLOBAL_STEP: 96550 + | > loss_text_ce: 0.029481086879968643 (0.03217030255300335) + | > loss_mel_ce: 3.1857011318206787 (3.086732162150385) + | > loss: 0.803795576095581 (0.7797256162248029) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1961 (0.2446425692190675) + | > loader_time: 0.0116 (0.01157631921263443) + + + --> TIME: 2024-10-20 10:14:02 -- STEP: 27541/69059 -- GLOBAL_STEP: 96600 + | > loss_text_ce: 0.031099455431103706 (0.03216900756348405) + | > loss_mel_ce: 2.891777753829956 (3.0866442980717173) + | > loss: 0.7307193279266357 (0.7797033264570247) + | > current_lr: 5e-06 + | > step_time: 0.1871 (0.24459043242028375) + | > loader_time: 0.0109 (0.011575673435998335) + + + --> TIME: 2024-10-20 10:14:21 -- STEP: 27591/69059 -- GLOBAL_STEP: 96650 + | > loss_text_ce: 0.030731379985809326 (0.032168373533789756) + | > loss_mel_ce: 3.2511048316955566 (3.0865943601739563) + | > loss: 0.8204590678215027 (0.7796906834669892) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1902 (0.24460628004082052) + | > loader_time: 0.0106 (0.011575899180831599) + + + --> TIME: 2024-10-20 10:14:39 -- STEP: 27641/69059 -- GLOBAL_STEP: 96700 + | > loss_text_ce: 0.03265721723437309 (0.0321680374258486) + | > loss_mel_ce: 2.920475482940674 (3.086610105645691) + | > loss: 0.7382831573486328 (0.7796945358093813) + | > current_lr: 5e-06 + | > step_time: 0.1785 (0.24459793269990057) + | > loader_time: 0.0101 (0.011575798943316963) + + + --> TIME: 2024-10-20 10:14:57 -- STEP: 27691/69059 -- GLOBAL_STEP: 96750 + | > loss_text_ce: 0.030663765966892242 (0.032167157819944885) + | > loss_mel_ce: 3.0078225135803223 (3.0864742960772524) + | > loss: 0.7596215605735779 (0.7796603635113318) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.335 (0.24457236383447828) + | > loader_time: 0.0118 (0.011575405683183736) + + + --> TIME: 2024-10-20 10:15:15 -- STEP: 27741/69059 -- GLOBAL_STEP: 96800 + | > loss_text_ce: 0.0316939614713192 (0.0321665644183873) + | > loss_mel_ce: 3.228440046310425 (3.08651258445181) + | > loss: 0.815033495426178 (0.7796697872562777) + | > current_lr: 5e-06 + | > step_time: 0.2804 (0.24456126017567237) + | > loader_time: 0.0119 (0.01157469870804288) + + + --> TIME: 2024-10-20 10:15:35 -- STEP: 27791/69059 -- GLOBAL_STEP: 96850 + | > loss_text_ce: 0.03142470121383667 (0.032167297660519394) + | > loss_mel_ce: 2.677415609359741 (3.0865424751896415) + | > loss: 0.6772100925445557 (0.7796774432489368) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2154 (0.2445994073663207) + | > loader_time: 0.011 (0.011574638155381655) + + + --> TIME: 2024-10-20 10:15:54 -- STEP: 27841/69059 -- GLOBAL_STEP: 96900 + | > loss_text_ce: 0.03207729384303093 (0.0321667380768789) + | > loss_mel_ce: 2.8253047466278076 (3.0865430200602075) + | > loss: 0.7143455147743225 (0.7796774395820432) + | > current_lr: 5e-06 + | > step_time: 0.1988 (0.24462292046302195) + | > loader_time: 0.0141 (0.011575412435104467) + + + --> TIME: 2024-10-20 10:16:12 -- STEP: 27891/69059 -- GLOBAL_STEP: 96950 + | > loss_text_ce: 0.03489784523844719 (0.03216585013752966) + | > loss_mel_ce: 3.0255930423736572 (3.0864662896957977) + | > loss: 0.7651227116584778 (0.7796580350085722) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1978 (0.244592813595631) + | > loader_time: 0.0102 (0.011574599755797228) + + + --> TIME: 2024-10-20 10:16:30 -- STEP: 27941/69059 -- GLOBAL_STEP: 97000 + | > loss_text_ce: 0.032966312021017075 (0.03216561089646086) + | > loss_mel_ce: 3.0048604011535645 (3.086418031711811) + | > loss: 0.7594566941261292 (0.7796459107015685) + | > current_lr: 5e-06 + | > step_time: 0.215 (0.24458749674593674) + | > loader_time: 0.0111 (0.01157909115273577) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_97000.pth + + --> TIME: 2024-10-20 10:17:03 -- STEP: 27991/69059 -- GLOBAL_STEP: 97050 + | > loss_text_ce: 0.031467314809560776 (0.03216563782959696) + | > loss_mel_ce: 3.1401987075805664 (3.086429038882) + | > loss: 0.792916476726532 (0.7796486692282764) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3839 (0.24458479742448327) + | > loader_time: 0.0123 (0.01157842055168069) + + + --> TIME: 2024-10-20 10:17:20 -- STEP: 28041/69059 -- GLOBAL_STEP: 97100 + | > loss_text_ce: 0.03115774877369404 (0.032164087405129306) + | > loss_mel_ce: 3.082611083984375 (3.0862798357667858) + | > loss: 0.7784422039985657 (0.7796109808434653) + | > current_lr: 5e-06 + | > step_time: 0.1853 (0.24455985389172938) + | > loader_time: 0.0105 (0.011577576085319832) + + + --> TIME: 2024-10-20 10:17:40 -- STEP: 28091/69059 -- GLOBAL_STEP: 97150 + | > loss_text_ce: 0.03204986825585365 (0.0321629519783311) + | > loss_mel_ce: 3.208186149597168 (3.08627225268769) + | > loss: 0.8100590109825134 (0.7796088012136196) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.3107 (0.244578205954799) + | > loader_time: 0.0116 (0.011578018415530562) + + + --> TIME: 2024-10-20 10:18:00 -- STEP: 28141/69059 -- GLOBAL_STEP: 97200 + | > loss_text_ce: 0.031448617577552795 (0.03216228016436472) + | > loss_mel_ce: 2.860395908355713 (3.086304280207278) + | > loss: 0.7229611277580261 (0.7796166401390477) + | > current_lr: 5e-06 + | > step_time: 0.187 (0.24462833242212037) + | > loader_time: 0.0112 (0.011578316568400596) + + + --> TIME: 2024-10-20 10:18:20 -- STEP: 28191/69059 -- GLOBAL_STEP: 97250 + | > loss_text_ce: 0.03366992995142937 (0.03216233081328845) + | > loss_mel_ce: 2.6812474727630615 (3.08631627745974) + | > loss: 0.6787293553352356 (0.7796196521167902) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.216 (0.24466373859463844) + | > loader_time: 0.0108 (0.011578402189900423) + + + --> TIME: 2024-10-20 10:18:37 -- STEP: 28241/69059 -- GLOBAL_STEP: 97300 + | > loss_text_ce: 0.030668023973703384 (0.032161050772570005) + | > loss_mel_ce: 2.948239326477051 (3.086146174601335) + | > loss: 0.744726836681366 (0.7795768063980407) + | > current_lr: 5e-06 + | > step_time: 0.3235 (0.24463674525803297) + | > loader_time: 0.0123 (0.011578129251399802) + + + --> TIME: 2024-10-20 10:18:55 -- STEP: 28291/69059 -- GLOBAL_STEP: 97350 + | > loss_text_ce: 0.029543692246079445 (0.03216096074639187) + | > loss_mel_ce: 3.2262117862701416 (3.086094570869484) + | > loss: 0.8139388561248779 (0.7795638829592928) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2042 (0.24461612877210853) + | > loader_time: 0.011 (0.011578113250433083) + + + --> TIME: 2024-10-20 10:19:14 -- STEP: 28341/69059 -- GLOBAL_STEP: 97400 + | > loss_text_ce: 0.032006777822971344 (0.03216079949266868) + | > loss_mel_ce: 2.8168859481811523 (3.086052339249692) + | > loss: 0.7122231721878052 (0.7795532847387955) + | > current_lr: 5e-06 + | > step_time: 0.3805 (0.24463565769951343) + | > loader_time: 0.0108 (0.01157787932130134) + + + --> TIME: 2024-10-20 10:19:33 -- STEP: 28391/69059 -- GLOBAL_STEP: 97450 + | > loss_text_ce: 0.033443842083215714 (0.03216020970992613) + | > loss_mel_ce: 3.305014133453369 (3.086051502013126) + | > loss: 0.8346145153045654 (0.7795529279862201) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.324 (0.24462953763809095) + | > loader_time: 0.0128 (0.011578030242105684) + + + --> TIME: 2024-10-20 10:19:52 -- STEP: 28441/69059 -- GLOBAL_STEP: 97500 + | > loss_text_ce: 0.03270857408642769 (0.03215988369503428) + | > loss_mel_ce: 3.200754404067993 (3.0860515117796035) + | > loss: 0.8083657622337341 (0.7795528489210227) + | > current_lr: 5e-06 + | > step_time: 0.2771 (0.2446432762317679) + | > loader_time: 0.0117 (0.011578254519306632) + + + --> TIME: 2024-10-20 10:20:09 -- STEP: 28491/69059 -- GLOBAL_STEP: 97550 + | > loss_text_ce: 0.030499015003442764 (0.03215912713020265) + | > loss_mel_ce: 2.809318780899048 (3.086095733283737) + | > loss: 0.7099544405937195 (0.7795637151641083) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1984 (0.24461503455815545) + | > loader_time: 0.0109 (0.011577631673993286) + + + --> TIME: 2024-10-20 10:20:26 -- STEP: 28541/69059 -- GLOBAL_STEP: 97600 + | > loss_text_ce: 0.03447908163070679 (0.03215851402466532) + | > loss_mel_ce: 3.1589183807373047 (3.0860423425924455) + | > loss: 0.7983493804931641 (0.7795502142145011) + | > current_lr: 5e-06 + | > step_time: 0.1674 (0.2445760383731595) + | > loader_time: 0.0104 (0.011577927963087362) + + + --> TIME: 2024-10-20 10:20:45 -- STEP: 28591/69059 -- GLOBAL_STEP: 97650 + | > loss_text_ce: 0.03364568203687668 (0.032157953265759924) + | > loss_mel_ce: 3.21637225151062 (3.085978110103162) + | > loss: 0.8125044703483582 (0.7795340159015994) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.4336 (0.24458068051988138) + | > loader_time: 0.0125 (0.011578180186958898) + + + --> TIME: 2024-10-20 10:21:05 -- STEP: 28641/69059 -- GLOBAL_STEP: 97700 + | > loss_text_ce: 0.031076529994606972 (0.03215745205807461) + | > loss_mel_ce: 3.1671438217163086 (3.0860106472878863) + | > loss: 0.7995550632476807 (0.7795420248978528) + | > current_lr: 5e-06 + | > step_time: 0.3495 (0.24461494376675133) + | > loader_time: 0.0126 (0.011578415938626747) + + + --> TIME: 2024-10-20 10:21:24 -- STEP: 28691/69059 -- GLOBAL_STEP: 97750 + | > loss_text_ce: 0.03488251939415932 (0.03215718382907742) + | > loss_mel_ce: 3.135904550552368 (3.0860392603253004) + | > loss: 0.7926967740058899 (0.7795491110933256) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2585 (0.24463915082818444) + | > loader_time: 0.0119 (0.011578804700979814) + + + --> TIME: 2024-10-20 10:21:43 -- STEP: 28741/69059 -- GLOBAL_STEP: 97800 + | > loss_text_ce: 0.029284605756402016 (0.032155760709047215) + | > loss_mel_ce: 3.078406572341919 (3.0859498682261215) + | > loss: 0.7769228219985962 (0.779526407289951) + | > current_lr: 5e-06 + | > step_time: 0.2052 (0.24464499335303774) + | > loader_time: 0.0115 (0.011578995592264572) + + + --> TIME: 2024-10-20 10:22:02 -- STEP: 28791/69059 -- GLOBAL_STEP: 97850 + | > loss_text_ce: 0.02933092601597309 (0.0321554018515199) + | > loss_mel_ce: 2.8750431537628174 (3.0858700616184613) + | > loss: 0.7260935306549072 (0.7795063659177665) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.1959 (0.24466749401257493) + | > loader_time: 0.0106 (0.011579260208844304) + + + --> TIME: 2024-10-20 10:22:21 -- STEP: 28841/69059 -- GLOBAL_STEP: 97900 + | > loss_text_ce: 0.029837118461728096 (0.03215477384354565) + | > loss_mel_ce: 2.9301819801330566 (3.0858936962466696) + | > loss: 0.7400047779083252 (0.7795121175671352) + | > current_lr: 5e-06 + | > step_time: 0.2149 (0.2446672128104192) + | > loader_time: 0.0112 (0.011579004390027844) + + + --> TIME: 2024-10-20 10:22:38 -- STEP: 28891/69059 -- GLOBAL_STEP: 97950 + | > loss_text_ce: 0.03213130682706833 (0.03215352803167477) + | > loss_mel_ce: 3.2002668380737305 (3.085833388925317) + | > loss: 0.8080995082855225 (0.7794967292769666) + | > grad_norm: 0 (0.0) + | > current_lr: 5e-06 + | > step_time: 0.2295 (0.24463110063179264) + | > loader_time: 0.0184 (0.011578549312483153) + + + --> TIME: 2024-10-20 10:22:57 -- STEP: 28941/69059 -- GLOBAL_STEP: 98000 + | > loss_text_ce: 0.03160318732261658 (0.0321524259394734) + | > loss_mel_ce: 3.0817275047302246 (3.085763417604741) + | > loss: 0.7783326506614685 (0.7794789609185421) + | > current_lr: 5e-06 + | > step_time: 0.1863 (0.24463708790140865) + | > loader_time: 0.0105 (0.011578174028040393) + + + > CHECKPOINT : checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b/checkpoint_98000.pth + ! Run is kept in checkpoints/GPT_XTTS_FT-October-19-2024_10+47PM-1a53d9b