| > Training Environment: | |
| | > Backend: Torch | |
| | > Mixed precision: True | |
| | > Precision: fp16 | |
| | > Current device: 0 | |
| | > Num. of GPUs: 1 | |
| | > Num. of CPUs: 24 | |
| | > Num. of Torch Threads: 24 | |
| | > Torch seed: 54321 | |
| | > Torch CUDNN: True | |
| | > Torch CUDNN deterministic: False | |
| | > Torch CUDNN benchmark: True | |
| | > Torch TF32 MatMul: False | |
| > Start Tensorboard: tensorboard --logdir=/home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| > Restoring from checkpoint.pth ... | |
| > Restoring Model... | |
| > Partial model initialization... | |
| | > Layer dimention missmatch between model definition and checkpoint: text_encoder.emb.weight | |
| | > 948 / 949 layers are restored. | |
| > Model restored from step 826000 | |
| > Model has 83060908 parameters | |
| [4m[1m > EPOCH: 0/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:05:23) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time: 0.0025544166564941406 [0m(+0) | |
| | > avg_loss_disc: 2.579561471939087 [0m(+0) | |
| | > avg_loss_disc_real_0: 0.22504794597625732 [0m(+0) | |
| | > avg_loss_disc_real_1: 0.23633995652198792 [0m(+0) | |
| | > avg_loss_disc_real_2: 0.23231595754623413 [0m(+0) | |
| | > avg_loss_disc_real_3: 0.24025149643421173 [0m(+0) | |
| | > avg_loss_disc_real_4: 0.18514925241470337 [0m(+0) | |
| | > avg_loss_disc_real_5: 0.22825410962104797 [0m(+0) | |
| | > avg_loss_0: 2.579561471939087 [0m(+0) | |
| | > avg_loss_gen: 2.095607280731201 [0m(+0) | |
| | > avg_loss_kl: 2.954455852508545 [0m(+0) | |
| | > avg_loss_feat: 6.803194046020508 [0m(+0) | |
| | > avg_loss_mel: 16.120973587036133 [0m(+0) | |
| | > avg_loss_duration: 1.8933916091918945 [0m(+0) | |
| | > avg_loss_1: 29.86762237548828 [0m(+0) | |
| > BEST MODEL : /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000/best_model_826027.pth | |
| [4m[1m > EPOCH: 1/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:06:29) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0036211013793945312 [0m(+0.0010666847229003906) | |
| | > avg_loss_disc:[92m 2.360954523086548 [0m(-0.21860694885253906) | |
| | > avg_loss_disc_real_0:[92m 0.11433742940425873 [0m(-0.1107105165719986) | |
| | > avg_loss_disc_real_1:[92m 0.11346497386693954 [0m(-0.12287498265504837) | |
| | > avg_loss_disc_real_2:[92m 0.14866462349891663 [0m(-0.0836513340473175) | |
| | > avg_loss_disc_real_3:[91m 0.2403942048549652 [0m(+0.000142708420753479) | |
| | > avg_loss_disc_real_4:[91m 0.24388839304447174 [0m(+0.05873914062976837) | |
| | > avg_loss_disc_real_5:[92m 0.19841952621936798 [0m(-0.029834583401679993) | |
| | > avg_loss_0:[92m 2.360954523086548 [0m(-0.21860694885253906) | |
| | > avg_loss_gen:[91m 2.211665153503418 [0m(+0.1160578727722168) | |
| | > avg_loss_kl:[92m 2.9490206241607666 [0m(-0.00543522834777832) | |
| | > avg_loss_feat:[91m 8.537314414978027 [0m(+1.7341203689575195) | |
| | > avg_loss_mel:[91m 22.604536056518555 [0m(+6.483562469482422) | |
| | > avg_loss_duration:[91m 1.9063528776168823 [0m(+0.012961268424987793) | |
| | > avg_loss_1:[91m 38.20888900756836 [0m(+8.341266632080078) | |
| [4m[1m > EPOCH: 2/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:06:51) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0029449462890625 [0m(-0.0006761550903320312) | |
| | > avg_loss_disc:[91m 2.836214780807495 [0m(+0.47526025772094727) | |
| | > avg_loss_disc_real_0:[91m 0.18877620995044708 [0m(+0.07443878054618835) | |
| | > avg_loss_disc_real_1:[91m 0.23450841009616852 [0m(+0.12104343622922897) | |
| | > avg_loss_disc_real_2:[91m 0.17941533029079437 [0m(+0.030750706791877747) | |
| | > avg_loss_disc_real_3:[92m 0.21590179204940796 [0m(-0.02449241280555725) | |
| | > avg_loss_disc_real_4:[91m 0.2847995460033417 [0m(+0.040911152958869934) | |
| | > avg_loss_disc_real_5:[91m 0.20625822246074677 [0m(+0.007838696241378784) | |
| | > avg_loss_0:[91m 2.836214780807495 [0m(+0.47526025772094727) | |
| | > avg_loss_gen:[92m 1.9285584688186646 [0m(-0.2831066846847534) | |
| | > avg_loss_kl:[91m 2.9806227684020996 [0m(+0.03160214424133301) | |
| | > avg_loss_feat:[92m 7.743529319763184 [0m(-0.7937850952148438) | |
| | > avg_loss_mel:[92m 17.637630462646484 [0m(-4.96690559387207) | |
| | > avg_loss_duration:[92m 1.8946624994277954 [0m(-0.011690378189086914) | |
| | > avg_loss_1:[92m 32.185001373291016 [0m(-6.023887634277344) | |
| [4m[1m > EPOCH: 3/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:07:10) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0026216506958007812 [0m(-0.00032329559326171875) | |
| | > avg_loss_disc:[92m 2.4784274101257324 [0m(-0.3577873706817627) | |
| | > avg_loss_disc_real_0:[92m 0.14986148476600647 [0m(-0.03891472518444061) | |
| | > avg_loss_disc_real_1:[92m 0.1897958517074585 [0m(-0.04471255838871002) | |
| | > avg_loss_disc_real_2:[92m 0.16911624372005463 [0m(-0.010299086570739746) | |
| | > avg_loss_disc_real_3:[91m 0.25678882002830505 [0m(+0.040887027978897095) | |
| | > avg_loss_disc_real_4:[92m 0.16667822003364563 [0m(-0.11812132596969604) | |
| | > avg_loss_disc_real_5:[92m 0.11786417663097382 [0m(-0.08839404582977295) | |
| | > avg_loss_0:[92m 2.4784274101257324 [0m(-0.3577873706817627) | |
| | > avg_loss_gen:[91m 2.258392333984375 [0m(+0.32983386516571045) | |
| | > avg_loss_kl:[92m 2.243771553039551 [0m(-0.7368512153625488) | |
| | > avg_loss_feat:[91m 10.215510368347168 [0m(+2.4719810485839844) | |
| | > avg_loss_mel:[91m 20.45659065246582 [0m(+2.818960189819336) | |
| | > avg_loss_duration:[92m 1.8232922554016113 [0m(-0.07137024402618408) | |
| | > avg_loss_1:[91m 36.997554779052734 [0m(+4.812553405761719) | |
| [4m[1m > EPOCH: 4/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:07:30) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.002149343490600586 [0m(-0.0004723072052001953) | |
| | > avg_loss_disc:[91m 2.8351030349731445 [0m(+0.3566756248474121) | |
| | > avg_loss_disc_real_0:[91m 0.202328160405159 [0m(+0.05246667563915253) | |
| | > avg_loss_disc_real_1:[91m 0.19721390306949615 [0m(+0.007418051362037659) | |
| | > avg_loss_disc_real_2:[91m 0.19663220643997192 [0m(+0.027515962719917297) | |
| | > avg_loss_disc_real_3:[92m 0.19390971958637238 [0m(-0.06287910044193268) | |
| | > avg_loss_disc_real_4:[91m 0.2577243447303772 [0m(+0.09104612469673157) | |
| | > avg_loss_disc_real_5:[91m 0.3611280024051666 [0m(+0.2432638257741928) | |
| | > avg_loss_0:[91m 2.8351030349731445 [0m(+0.3566756248474121) | |
| | > avg_loss_gen:[92m 2.100310802459717 [0m(-0.1580815315246582) | |
| | > avg_loss_kl:[91m 2.5330567359924316 [0m(+0.28928518295288086) | |
| | > avg_loss_feat:[92m 7.338407039642334 [0m(-2.877103328704834) | |
| | > avg_loss_mel:[92m 19.65290641784668 [0m(-0.8036842346191406) | |
| | > avg_loss_duration:[92m 1.7398550510406494 [0m(-0.08343720436096191) | |
| | > avg_loss_1:[92m 33.36453628540039 [0m(-3.6330184936523438) | |
| [4m[1m > EPOCH: 5/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:07:48) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.003258228302001953 [0m(+0.0011088848114013672) | |
| | > avg_loss_disc:[91m 2.8697474002838135 [0m(+0.034644365310668945) | |
| | > avg_loss_disc_real_0:[92m 0.1347222477197647 [0m(-0.06760591268539429) | |
| | > avg_loss_disc_real_1:[92m 0.15822717547416687 [0m(-0.038986727595329285) | |
| | > avg_loss_disc_real_2:[92m 0.1845826804637909 [0m(-0.01204952597618103) | |
| | > avg_loss_disc_real_3:[91m 0.2523535192012787 [0m(+0.05844379961490631) | |
| | > avg_loss_disc_real_4:[92m 0.14204007387161255 [0m(-0.11568427085876465) | |
| | > avg_loss_disc_real_5:[92m 0.1628914326429367 [0m(-0.19823656976222992) | |
| | > avg_loss_0:[91m 2.8697474002838135 [0m(+0.034644365310668945) | |
| | > avg_loss_gen:[92m 1.76955246925354 [0m(-0.33075833320617676) | |
| | > avg_loss_kl:[92m 1.7157461643218994 [0m(-0.8173105716705322) | |
| | > avg_loss_feat:[91m 9.666239738464355 [0m(+2.3278326988220215) | |
| | > avg_loss_mel:[91m 21.284114837646484 [0m(+1.6312084197998047) | |
| | > avg_loss_duration:[92m 1.681728720664978 [0m(-0.05812633037567139) | |
| | > avg_loss_1:[91m 36.11738204956055 [0m(+2.7528457641601562) | |
| [4m[1m > EPOCH: 6/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:08:08) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0019330978393554688 [0m(-0.0013251304626464844) | |
| | > avg_loss_disc:[92m 2.5998082160949707 [0m(-0.2699391841888428) | |
| | > avg_loss_disc_real_0:[92m 0.10893561691045761 [0m(-0.0257866308093071) | |
| | > avg_loss_disc_real_1:[91m 0.21296951174736023 [0m(+0.05474233627319336) | |
| | > avg_loss_disc_real_2:[91m 0.22080537676811218 [0m(+0.03622269630432129) | |
| | > avg_loss_disc_real_3:[92m 0.1968725025653839 [0m(-0.055481016635894775) | |
| | > avg_loss_disc_real_4:[91m 0.3077707290649414 [0m(+0.16573065519332886) | |
| | > avg_loss_disc_real_5:[92m 0.11337371170520782 [0m(-0.04951772093772888) | |
| | > avg_loss_0:[92m 2.5998082160949707 [0m(-0.2699391841888428) | |
| | > avg_loss_gen:[91m 2.1091349124908447 [0m(+0.3395824432373047) | |
| | > avg_loss_kl:[91m 2.2263669967651367 [0m(+0.5106208324432373) | |
| | > avg_loss_feat:[91m 11.159883499145508 [0m(+1.4936437606811523) | |
| | > avg_loss_mel:[91m 21.78208351135254 [0m(+0.4979686737060547) | |
| | > avg_loss_duration:[92m 1.6071841716766357 [0m(-0.07454454898834229) | |
| | > avg_loss_1:[91m 38.8846549987793 [0m(+2.76727294921875) | |
| [4m[1m > EPOCH: 7/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:08:29) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0019304752349853516 [0m(-2.6226043701171875e-06) | |
| | > avg_loss_disc:[92m 2.4500019550323486 [0m(-0.14980626106262207) | |
| | > avg_loss_disc_real_0:[92m 0.09126907587051392 [0m(-0.017666541039943695) | |
| | > avg_loss_disc_real_1:[92m 0.13244855403900146 [0m(-0.08052095770835876) | |
| | > avg_loss_disc_real_2:[92m 0.116974376142025 [0m(-0.10383100062608719) | |
| | > avg_loss_disc_real_3:[92m 0.13961941003799438 [0m(-0.057253092527389526) | |
| | > avg_loss_disc_real_4:[92m 0.1451183557510376 [0m(-0.1626523733139038) | |
| | > avg_loss_disc_real_5:[91m 0.11358281224966049 [0m(+0.00020910054445266724) | |
| | > avg_loss_0:[92m 2.4500019550323486 [0m(-0.14980626106262207) | |
| | > avg_loss_gen:[92m 1.6871346235275269 [0m(-0.42200028896331787) | |
| | > avg_loss_kl:[92m 2.055893659591675 [0m(-0.17047333717346191) | |
| | > avg_loss_feat:[91m 12.322132110595703 [0m(+1.1622486114501953) | |
| | > avg_loss_mel:[91m 22.194091796875 [0m(+0.41200828552246094) | |
| | > avg_loss_duration:[91m 1.6218042373657227 [0m(+0.014620065689086914) | |
| | > avg_loss_1:[91m 39.88105392456055 [0m(+0.99639892578125) | |
| [4m[1m > EPOCH: 8/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:08:49) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0020303726196289062 [0m(+9.989738464355469e-05) | |
| | > avg_loss_disc:[91m 2.458998680114746 [0m(+0.008996725082397461) | |
| | > avg_loss_disc_real_0:[91m 0.1939530372619629 [0m(+0.10268396139144897) | |
| | > avg_loss_disc_real_1:[91m 0.18585415184497833 [0m(+0.05340559780597687) | |
| | > avg_loss_disc_real_2:[91m 0.22415761649608612 [0m(+0.10718324035406113) | |
| | > avg_loss_disc_real_3:[91m 0.23032917082309723 [0m(+0.09070976078510284) | |
| | > avg_loss_disc_real_4:[91m 0.256417453289032 [0m(+0.11129909753799438) | |
| | > avg_loss_disc_real_5:[91m 0.22092194855213165 [0m(+0.10733913630247116) | |
| | > avg_loss_0:[91m 2.458998680114746 [0m(+0.008996725082397461) | |
| | > avg_loss_gen:[91m 2.281503200531006 [0m(+0.594368577003479) | |
| | > avg_loss_kl:[91m 2.1694588661193848 [0m(+0.11356520652770996) | |
| | > avg_loss_feat:[92m 6.771675109863281 [0m(-5.550457000732422) | |
| | > avg_loss_mel:[92m 19.732463836669922 [0m(-2.461627960205078) | |
| | > avg_loss_duration:[92m 1.5910258293151855 [0m(-0.03077840805053711) | |
| | > avg_loss_1:[92m 32.54612731933594 [0m(-7.334926605224609) | |
| [4m[1m > EPOCH: 9/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:09:12) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.002134561538696289 [0m(+0.00010418891906738281) | |
| | > avg_loss_disc:[92m 2.3202075958251953 [0m(-0.13879108428955078) | |
| | > avg_loss_disc_real_0:[92m 0.1641819328069687 [0m(-0.0297711044549942) | |
| | > avg_loss_disc_real_1:[91m 0.2069983035326004 [0m(+0.02114415168762207) | |
| | > avg_loss_disc_real_2:[92m 0.19238199293613434 [0m(-0.03177562355995178) | |
| | > avg_loss_disc_real_3:[92m 0.21948115527629852 [0m(-0.010848015546798706) | |
| | > avg_loss_disc_real_4:[92m 0.20658878982067108 [0m(-0.0498286634683609) | |
| | > avg_loss_disc_real_5:[92m 0.21016131341457367 [0m(-0.010760635137557983) | |
| | > avg_loss_0:[92m 2.3202075958251953 [0m(-0.13879108428955078) | |
| | > avg_loss_gen:[91m 2.370643138885498 [0m(+0.08913993835449219) | |
| | > avg_loss_kl:[92m 1.8640230894088745 [0m(-0.30543577671051025) | |
| | > avg_loss_feat:[91m 7.0724196434021 [0m(+0.30074453353881836) | |
| | > avg_loss_mel:[91m 20.123937606811523 [0m(+0.39147377014160156) | |
| | > avg_loss_duration:[92m 1.5861397981643677 [0m(-0.004886031150817871) | |
| | > avg_loss_1:[91m 33.01716232299805 [0m(+0.4710350036621094) | |
| [4m[1m > EPOCH: 10/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:09:34) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0019383430480957031 [0m(-0.00019621849060058594) | |
| | > avg_loss_disc:[91m 2.6848320960998535 [0m(+0.3646245002746582) | |
| | > avg_loss_disc_real_0:[92m 0.1019434928894043 [0m(-0.06223843991756439) | |
| | > avg_loss_disc_real_1:[91m 0.22225375473499298 [0m(+0.015255451202392578) | |
| | > avg_loss_disc_real_2:[91m 0.29437896609306335 [0m(+0.10199697315692902) | |
| | > avg_loss_disc_real_3:[91m 0.369914710521698 [0m(+0.15043355524539948) | |
| | > avg_loss_disc_real_4:[91m 0.2501886785030365 [0m(+0.04359988868236542) | |
| | > avg_loss_disc_real_5:[91m 0.2736251652240753 [0m(+0.06346385180950165) | |
| | > avg_loss_0:[91m 2.6848320960998535 [0m(+0.3646245002746582) | |
| | > avg_loss_gen:[92m 2.2344932556152344 [0m(-0.13614988327026367) | |
| | > avg_loss_kl:[91m 2.1780905723571777 [0m(+0.3140674829483032) | |
| | > avg_loss_feat:[91m 8.309244155883789 [0m(+1.2368245124816895) | |
| | > avg_loss_mel:[92m 18.31509780883789 [0m(-1.8088397979736328) | |
| | > avg_loss_duration:[91m 1.5895346403121948 [0m(+0.0033948421478271484) | |
| | > avg_loss_1:[92m 32.626461029052734 [0m(-0.3907012939453125) | |
| [4m[1m > EPOCH: 11/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:09:59) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018219947814941406 [0m(-0.0001163482666015625) | |
| | > avg_loss_disc:[92m 2.6257660388946533 [0m(-0.059066057205200195) | |
| | > avg_loss_disc_real_0:[91m 0.21502818167209625 [0m(+0.11308468878269196) | |
| | > avg_loss_disc_real_1:[92m 0.17747360467910767 [0m(-0.044780150055885315) | |
| | > avg_loss_disc_real_2:[92m 0.17541557550430298 [0m(-0.11896339058876038) | |
| | > avg_loss_disc_real_3:[92m 0.18490779399871826 [0m(-0.18500691652297974) | |
| | > avg_loss_disc_real_4:[92m 0.17014773190021515 [0m(-0.08004094660282135) | |
| | > avg_loss_disc_real_5:[92m 0.16564661264419556 [0m(-0.10797855257987976) | |
| | > avg_loss_0:[92m 2.6257660388946533 [0m(-0.059066057205200195) | |
| | > avg_loss_gen:[92m 1.820864200592041 [0m(-0.41362905502319336) | |
| | > avg_loss_kl:[92m 2.026675224304199 [0m(-0.15141534805297852) | |
| | > avg_loss_feat:[91m 10.131511688232422 [0m(+1.8222675323486328) | |
| | > avg_loss_mel:[91m 20.05714988708496 [0m(+1.7420520782470703) | |
| | > avg_loss_duration:[92m 1.570576548576355 [0m(-0.018958091735839844) | |
| | > avg_loss_1:[91m 35.60677719116211 [0m(+2.980316162109375) | |
| [4m[1m > EPOCH: 12/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:10:21) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0019230842590332031 [0m(+0.0001010894775390625) | |
| | > avg_loss_disc:[91m 2.7327022552490234 [0m(+0.10693621635437012) | |
| | > avg_loss_disc_real_0:[91m 0.23379340767860413 [0m(+0.018765226006507874) | |
| | > avg_loss_disc_real_1:[91m 0.24418646097183228 [0m(+0.06671285629272461) | |
| | > avg_loss_disc_real_2:[91m 0.2957330346107483 [0m(+0.12031745910644531) | |
| | > avg_loss_disc_real_3:[91m 0.2668507397174835 [0m(+0.08194294571876526) | |
| | > avg_loss_disc_real_4:[91m 0.30550694465637207 [0m(+0.13535921275615692) | |
| | > avg_loss_disc_real_5:[91m 0.28343188762664795 [0m(+0.11778527498245239) | |
| | > avg_loss_0:[91m 2.7327022552490234 [0m(+0.10693621635437012) | |
| | > avg_loss_gen:[91m 2.1545863151550293 [0m(+0.3337221145629883) | |
| | > avg_loss_kl:[92m 1.9101665019989014 [0m(-0.11650872230529785) | |
| | > avg_loss_feat:[92m 3.6649274826049805 [0m(-6.466584205627441) | |
| | > avg_loss_mel:[92m 9.204965591430664 [0m(-10.852184295654297) | |
| | > avg_loss_duration:[92m 1.5653738975524902 [0m(-0.005202651023864746) | |
| | > avg_loss_1:[92m 18.50002098083496 [0m(-17.10675621032715) | |
| > BEST MODEL : /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000/best_model_826339.pth | |
| [4m[1m > EPOCH: 13/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:10:45) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.002056598663330078 [0m(+0.000133514404296875) | |
| | > avg_loss_disc:[92m 2.495265245437622 [0m(-0.23743700981140137) | |
| | > avg_loss_disc_real_0:[92m 0.1456478089094162 [0m(-0.08814559876918793) | |
| | > avg_loss_disc_real_1:[91m 0.29628562927246094 [0m(+0.05209916830062866) | |
| | > avg_loss_disc_real_2:[92m 0.23488187789916992 [0m(-0.06085115671157837) | |
| | > avg_loss_disc_real_3:[92m 0.24811452627182007 [0m(-0.018736213445663452) | |
| | > avg_loss_disc_real_4:[92m 0.2415287345647812 [0m(-0.06397821009159088) | |
| | > avg_loss_disc_real_5:[92m 0.23950818181037903 [0m(-0.04392370581626892) | |
| | > avg_loss_0:[92m 2.495265245437622 [0m(-0.23743700981140137) | |
| | > avg_loss_gen:[91m 2.332970380783081 [0m(+0.17838406562805176) | |
| | > avg_loss_kl:[91m 2.0096042156219482 [0m(+0.09943771362304688) | |
| | > avg_loss_feat:[91m 4.751073837280273 [0m(+1.086146354675293) | |
| | > avg_loss_mel:[91m 14.550800323486328 [0m(+5.345834732055664) | |
| | > avg_loss_duration:[92m 1.5579960346221924 [0m(-0.0073778629302978516) | |
| | > avg_loss_1:[91m 25.202444076538086 [0m(+6.702423095703125) | |
| [4m[1m > EPOCH: 14/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:11:08) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.001909017562866211 [0m(-0.0001475811004638672) | |
| | > avg_loss_disc:[91m 2.571105480194092 [0m(+0.07584023475646973) | |
| | > avg_loss_disc_real_0:[92m 0.14245551824569702 [0m(-0.0031922906637191772) | |
| | > avg_loss_disc_real_1:[92m 0.16970688104629517 [0m(-0.12657874822616577) | |
| | > avg_loss_disc_real_2:[92m 0.2034100443124771 [0m(-0.03147183358669281) | |
| | > avg_loss_disc_real_3:[92m 0.216097891330719 [0m(-0.032016634941101074) | |
| | > avg_loss_disc_real_4:[91m 0.28562402725219727 [0m(+0.04409529268741608) | |
| | > avg_loss_disc_real_5:[92m 0.16510680317878723 [0m(-0.0744013786315918) | |
| | > avg_loss_0:[91m 2.571105480194092 [0m(+0.07584023475646973) | |
| | > avg_loss_gen:[92m 1.963249683380127 [0m(-0.3697206974029541) | |
| | > avg_loss_kl:[91m 2.300732135772705 [0m(+0.29112792015075684) | |
| | > avg_loss_feat:[91m 7.8136515617370605 [0m(+3.062577724456787) | |
| | > avg_loss_mel:[91m 19.084020614624023 [0m(+4.533220291137695) | |
| | > avg_loss_duration:[92m 1.556723952293396 [0m(-0.0012720823287963867) | |
| | > avg_loss_1:[91m 32.71837615966797 [0m(+7.515932083129883) | |
| [4m[1m > EPOCH: 15/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:11:32) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0020017623901367188 [0m(+9.274482727050781e-05) | |
| | > avg_loss_disc:[91m 2.7033262252807617 [0m(+0.13222074508666992) | |
| | > avg_loss_disc_real_0:[91m 0.1850876808166504 [0m(+0.04263216257095337) | |
| | > avg_loss_disc_real_1:[91m 0.2569637894630432 [0m(+0.08725690841674805) | |
| | > avg_loss_disc_real_2:[91m 0.24873216450214386 [0m(+0.04532212018966675) | |
| | > avg_loss_disc_real_3:[91m 0.2321649193763733 [0m(+0.016067028045654297) | |
| | > avg_loss_disc_real_4:[92m 0.25387001037597656 [0m(-0.0317540168762207) | |
| | > avg_loss_disc_real_5:[91m 0.3422059416770935 [0m(+0.17709913849830627) | |
| | > avg_loss_0:[91m 2.7033262252807617 [0m(+0.13222074508666992) | |
| | > avg_loss_gen:[91m 2.466217517852783 [0m(+0.5029678344726562) | |
| | > avg_loss_kl:[91m 2.473677158355713 [0m(+0.1729450225830078) | |
| | > avg_loss_feat:[91m 8.864312171936035 [0m(+1.0506606101989746) | |
| | > avg_loss_mel:[91m 20.450681686401367 [0m(+1.3666610717773438) | |
| | > avg_loss_duration:[92m 1.5559325218200684 [0m(-0.0007914304733276367) | |
| | > avg_loss_1:[91m 35.810821533203125 [0m(+3.0924453735351562) | |
| [4m[1m > EPOCH: 16/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:11:55) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018572807312011719 [0m(-0.00014448165893554688) | |
| | > avg_loss_disc:[92m 2.526150703430176 [0m(-0.17717552185058594) | |
| | > avg_loss_disc_real_0:[91m 0.22012364864349365 [0m(+0.03503596782684326) | |
| | > avg_loss_disc_real_1:[92m 0.20115916430950165 [0m(-0.055804625153541565) | |
| | > avg_loss_disc_real_2:[92m 0.17505493760108948 [0m(-0.07367722690105438) | |
| | > avg_loss_disc_real_3:[92m 0.17389781773090363 [0m(-0.058267101645469666) | |
| | > avg_loss_disc_real_4:[91m 0.2565661072731018 [0m(+0.002696096897125244) | |
| | > avg_loss_disc_real_5:[92m 0.1770845651626587 [0m(-0.16512137651443481) | |
| | > avg_loss_0:[92m 2.526150703430176 [0m(-0.17717552185058594) | |
| | > avg_loss_gen:[92m 2.2745003700256348 [0m(-0.19171714782714844) | |
| | > avg_loss_kl:[92m 1.9097598791122437 [0m(-0.5639172792434692) | |
| | > avg_loss_feat:[92m 7.223072052001953 [0m(-1.641240119934082) | |
| | > avg_loss_mel:[92m 18.74079704284668 [0m(-1.7098846435546875) | |
| | > avg_loss_duration:[92m 1.5443365573883057 [0m(-0.011595964431762695) | |
| | > avg_loss_1:[92m 31.69246482849121 [0m(-4.118356704711914) | |
| [4m[1m > EPOCH: 17/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:12:18) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0019030570983886719 [0m(+4.57763671875e-05) | |
| | > avg_loss_disc:[92m 2.5047130584716797 [0m(-0.021437644958496094) | |
| | > avg_loss_disc_real_0:[91m 0.2906636595726013 [0m(+0.07054001092910767) | |
| | > avg_loss_disc_real_1:[91m 0.20504474639892578 [0m(+0.0038855820894241333) | |
| | > avg_loss_disc_real_2:[91m 0.18310394883155823 [0m(+0.00804901123046875) | |
| | > avg_loss_disc_real_3:[91m 0.25080615282058716 [0m(+0.07690833508968353) | |
| | > avg_loss_disc_real_4:[91m 0.3209937810897827 [0m(+0.06442767381668091) | |
| | > avg_loss_disc_real_5:[91m 0.24696961045265198 [0m(+0.06988504528999329) | |
| | > avg_loss_0:[92m 2.5047130584716797 [0m(-0.021437644958496094) | |
| | > avg_loss_gen:[91m 2.532362699508667 [0m(+0.2578623294830322) | |
| | > avg_loss_kl:[91m 2.36430025100708 [0m(+0.4545403718948364) | |
| | > avg_loss_feat:[92m 6.0262274742126465 [0m(-1.1968445777893066) | |
| | > avg_loss_mel:[91m 19.45924186706543 [0m(+0.71844482421875) | |
| | > avg_loss_duration:[92m 1.5199471712112427 [0m(-0.02438938617706299) | |
| | > avg_loss_1:[91m 31.90207862854004 [0m(+0.20961380004882812) | |
| [4m[1m > EPOCH: 18/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:12:43) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.001995563507080078 [0m(+9.250640869140625e-05) | |
| | > avg_loss_disc:[92m 2.2198214530944824 [0m(-0.28489160537719727) | |
| | > avg_loss_disc_real_0:[92m 0.13773468136787415 [0m(-0.15292897820472717) | |
| | > avg_loss_disc_real_1:[92m 0.1682843267917633 [0m(-0.036760419607162476) | |
| | > avg_loss_disc_real_2:[91m 0.19840839505195618 [0m(+0.01530444622039795) | |
| | > avg_loss_disc_real_3:[92m 0.21382148563861847 [0m(-0.03698466718196869) | |
| | > avg_loss_disc_real_4:[92m 0.24576376378536224 [0m(-0.07523001730442047) | |
| | > avg_loss_disc_real_5:[92m 0.21759817004203796 [0m(-0.029371440410614014) | |
| | > avg_loss_0:[92m 2.2198214530944824 [0m(-0.28489160537719727) | |
| | > avg_loss_gen:[91m 2.7460777759552 [0m(+0.2137150764465332) | |
| | > avg_loss_kl:[92m 1.839993953704834 [0m(-0.5243062973022461) | |
| | > avg_loss_feat:[91m 7.987682819366455 [0m(+1.9614553451538086) | |
| | > avg_loss_mel:[92m 19.310388565063477 [0m(-0.14885330200195312) | |
| | > avg_loss_duration:[91m 1.5255579948425293 [0m(+0.005610823631286621) | |
| | > avg_loss_1:[91m 33.40970230102539 [0m(+1.5076236724853516) | |
| [4m[1m > EPOCH: 19/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:13:09) [0m | |
| [1m --> TIME: 2024-03-31 10:13:13 -- STEP: 5/26 -- GLOBAL_STEP: 826500[0m | |
| | > loss_disc: 2.4146957397460938 (2.450313854217529) | |
| | > loss_disc_real_0: 0.14674012362957 (0.159730064868927) | |
| | > loss_disc_real_1: 0.1975105255842209 (0.21452410221099855) | |
| | > loss_disc_real_2: 0.1922743171453476 (0.22672945261001587) | |
| | > loss_disc_real_3: 0.1833949238061905 (0.20404482185840606) | |
| | > loss_disc_real_4: 0.15661051869392395 (0.20521345734596252) | |
| | > loss_disc_real_5: 0.1906307488679886 (0.2050049215555191) | |
| | > loss_0: 2.4146957397460938 (2.450313854217529) | |
| | > grad_norm_0: tensor(16.6246, device='cuda:0') (tensor(13.1455, device='cuda:0')) | |
| | > loss_gen: 2.416147470474243 (2.4157761573791503) | |
| | > loss_kl: 2.1608104705810547 (2.2007261753082275) | |
| | > loss_feat: 8.497369766235352 (8.68954849243164) | |
| | > loss_mel: 18.566688537597656 (18.833867263793945) | |
| | > loss_duration: 1.5417736768722534 (1.5525463581085206) | |
| | > amp_scaler: 256.0 (256.0) | |
| | > loss_1: 33.18278884887695 (33.692465209960936) | |
| | > grad_norm_1: tensor(125.5654, device='cuda:0') (tensor(142.0199, device='cuda:0')) | |
| | > current_lr_0: 0.00019952553399667344 | |
| | > current_lr_1: 0.00019952553399667344 | |
| | > step_time: 0.6896 (0.6795442581176758) | |
| | > loader_time: 0.0076 (0.0070612430572509766) | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018498897552490234 [0m(-0.0001456737518310547) | |
| | > avg_loss_disc:[91m 2.860830783843994 [0m(+0.6410093307495117) | |
| | > avg_loss_disc_real_0:[92m 0.07924967259168625 [0m(-0.0584850087761879) | |
| | > avg_loss_disc_real_1:[91m 0.16997618973255157 [0m(+0.001691862940788269) | |
| | > avg_loss_disc_real_2:[91m 0.19842827320098877 [0m(+1.9878149032592773e-05) | |
| | > avg_loss_disc_real_3:[92m 0.2056673765182495 [0m(-0.008154109120368958) | |
| | > avg_loss_disc_real_4:[91m 0.25505146384239197 [0m(+0.009287700057029724) | |
| | > avg_loss_disc_real_5:[91m 0.3589268624782562 [0m(+0.14132869243621826) | |
| | > avg_loss_0:[91m 2.860830783843994 [0m(+0.6410093307495117) | |
| | > avg_loss_gen:[92m 1.702101707458496 [0m(-1.043976068496704) | |
| | > avg_loss_kl:[91m 1.9191093444824219 [0m(+0.07911539077758789) | |
| | > avg_loss_feat:[92m 5.535099029541016 [0m(-2.4525837898254395) | |
| | > avg_loss_mel:[92m 17.000019073486328 [0m(-2.3103694915771484) | |
| | > avg_loss_duration:[91m 1.5454752445220947 [0m(+0.01991724967956543) | |
| | > avg_loss_1:[92m 27.701805114746094 [0m(-5.707897186279297) | |
| [4m[1m > EPOCH: 20/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:13:34) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0021533966064453125 [0m(+0.00030350685119628906) | |
| | > avg_loss_disc:[92m 2.2710814476013184 [0m(-0.5897493362426758) | |
| | > avg_loss_disc_real_0:[91m 0.09960463643074036 [0m(+0.020354963839054108) | |
| | > avg_loss_disc_real_1:[91m 0.19256524741649628 [0m(+0.022589057683944702) | |
| | > avg_loss_disc_real_2:[91m 0.20135581493377686 [0m(+0.002927541732788086) | |
| | > avg_loss_disc_real_3:[92m 0.18886281549930573 [0m(-0.016804561018943787) | |
| | > avg_loss_disc_real_4:[92m 0.23164379596710205 [0m(-0.023407667875289917) | |
| | > avg_loss_disc_real_5:[92m 0.2516683042049408 [0m(-0.10725855827331543) | |
| | > avg_loss_0:[92m 2.2710814476013184 [0m(-0.5897493362426758) | |
| | > avg_loss_gen:[91m 2.2265045642852783 [0m(+0.5244028568267822) | |
| | > avg_loss_kl:[91m 2.0785317420959473 [0m(+0.1594223976135254) | |
| | > avg_loss_feat:[91m 7.550324440002441 [0m(+2.015225410461426) | |
| | > avg_loss_mel:[91m 17.956676483154297 [0m(+0.9566574096679688) | |
| | > avg_loss_duration:[92m 1.5389719009399414 [0m(-0.00650334358215332) | |
| | > avg_loss_1:[91m 31.351009368896484 [0m(+3.6492042541503906) | |
| [4m[1m > EPOCH: 21/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:13:56) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0020716190338134766 [0m(-8.177757263183594e-05) | |
| | > avg_loss_disc:[91m 2.556429147720337 [0m(+0.28534770011901855) | |
| | > avg_loss_disc_real_0:[91m 0.12546581029891968 [0m(+0.02586117386817932) | |
| | > avg_loss_disc_real_1:[91m 0.2367255538702011 [0m(+0.044160306453704834) | |
| | > avg_loss_disc_real_2:[92m 0.17709685862064362 [0m(-0.02425895631313324) | |
| | > avg_loss_disc_real_3:[91m 0.2612573504447937 [0m(+0.07239453494548798) | |
| | > avg_loss_disc_real_4:[91m 0.2858666181564331 [0m(+0.054222822189331055) | |
| | > avg_loss_disc_real_5:[92m 0.16043631732463837 [0m(-0.09123198688030243) | |
| | > avg_loss_0:[91m 2.556429147720337 [0m(+0.28534770011901855) | |
| | > avg_loss_gen:[92m 1.996422290802002 [0m(-0.23008227348327637) | |
| | > avg_loss_kl:[91m 2.1214284896850586 [0m(+0.04289674758911133) | |
| | > avg_loss_feat:[91m 10.281383514404297 [0m(+2.7310590744018555) | |
| | > avg_loss_mel:[91m 19.507619857788086 [0m(+1.550943374633789) | |
| | > avg_loss_duration:[92m 1.5056617259979248 [0m(-0.0333101749420166) | |
| | > avg_loss_1:[91m 35.412513732910156 [0m(+4.061504364013672) | |
| [4m[1m > EPOCH: 22/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:14:22) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0019686222076416016 [0m(-0.000102996826171875) | |
| | > avg_loss_disc:[92m 2.432293176651001 [0m(-0.12413597106933594) | |
| | > avg_loss_disc_real_0:[91m 0.1689225435256958 [0m(+0.04345673322677612) | |
| | > avg_loss_disc_real_1:[92m 0.20391099154949188 [0m(-0.03281456232070923) | |
| | > avg_loss_disc_real_2:[91m 0.21518933773040771 [0m(+0.0380924791097641) | |
| | > avg_loss_disc_real_3:[92m 0.1842881739139557 [0m(-0.07696917653083801) | |
| | > avg_loss_disc_real_4:[92m 0.27542081475257874 [0m(-0.01044580340385437) | |
| | > avg_loss_disc_real_5:[91m 0.16442622244358063 [0m(+0.003989905118942261) | |
| | > avg_loss_0:[92m 2.432293176651001 [0m(-0.12413597106933594) | |
| | > avg_loss_gen:[91m 2.2132091522216797 [0m(+0.21678686141967773) | |
| | > avg_loss_kl:[92m 1.8254882097244263 [0m(-0.2959402799606323) | |
| | > avg_loss_feat:[92m 6.03596830368042 [0m(-4.245415210723877) | |
| | > avg_loss_mel:[92m 19.248849868774414 [0m(-0.2587699890136719) | |
| | > avg_loss_duration:[91m 1.5278148651123047 [0m(+0.022153139114379883) | |
| | > avg_loss_1:[92m 30.851329803466797 [0m(-4.561183929443359) | |
| [4m[1m > EPOCH: 23/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:14:46) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018432140350341797 [0m(-0.00012540817260742188) | |
| | > avg_loss_disc:[91m 2.521531105041504 [0m(+0.08923792839050293) | |
| | > avg_loss_disc_real_0:[91m 0.24499277770519257 [0m(+0.07607023417949677) | |
| | > avg_loss_disc_real_1:[91m 0.26089686155319214 [0m(+0.056985870003700256) | |
| | > avg_loss_disc_real_2:[92m 0.17814958095550537 [0m(-0.037039756774902344) | |
| | > avg_loss_disc_real_3:[91m 0.22235319018363953 [0m(+0.03806501626968384) | |
| | > avg_loss_disc_real_4:[92m 0.2719460129737854 [0m(-0.003474801778793335) | |
| | > avg_loss_disc_real_5:[91m 0.2647368311882019 [0m(+0.10031060874462128) | |
| | > avg_loss_0:[91m 2.521531105041504 [0m(+0.08923792839050293) | |
| | > avg_loss_gen:[91m 2.6347410678863525 [0m(+0.42153191566467285) | |
| | > avg_loss_kl:[91m 2.1445794105529785 [0m(+0.31909120082855225) | |
| | > avg_loss_feat:[91m 9.143684387207031 [0m(+3.1077160835266113) | |
| | > avg_loss_mel:[92m 18.559925079345703 [0m(-0.6889247894287109) | |
| | > avg_loss_duration:[92m 1.5272908210754395 [0m(-0.0005240440368652344) | |
| | > avg_loss_1:[91m 34.01021957397461 [0m(+3.1588897705078125) | |
| [4m[1m > EPOCH: 24/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:15:10) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0021209716796875 [0m(+0.0002777576446533203) | |
| | > avg_loss_disc:[92m 2.410466194152832 [0m(-0.11106491088867188) | |
| | > avg_loss_disc_real_0:[92m 0.08595563471317291 [0m(-0.15903714299201965) | |
| | > avg_loss_disc_real_1:[92m 0.17231160402297974 [0m(-0.0885852575302124) | |
| | > avg_loss_disc_real_2:[91m 0.2597108781337738 [0m(+0.08156129717826843) | |
| | > avg_loss_disc_real_3:[91m 0.2549273371696472 [0m(+0.03257414698600769) | |
| | > avg_loss_disc_real_4:[92m 0.19602034986019135 [0m(-0.07592566311359406) | |
| | > avg_loss_disc_real_5:[91m 0.26538583636283875 [0m(+0.0006490051746368408) | |
| | > avg_loss_0:[92m 2.410466194152832 [0m(-0.11106491088867188) | |
| | > avg_loss_gen:[92m 2.4343385696411133 [0m(-0.20040249824523926) | |
| | > avg_loss_kl:[91m 2.1930034160614014 [0m(+0.04842400550842285) | |
| | > avg_loss_feat:[92m 8.565384864807129 [0m(-0.5782995223999023) | |
| | > avg_loss_mel:[91m 19.013505935668945 [0m(+0.4535808563232422) | |
| | > avg_loss_duration:[91m 1.5361652374267578 [0m(+0.00887441635131836) | |
| | > avg_loss_1:[92m 33.742401123046875 [0m(-0.2678184509277344) | |
| [4m[1m > EPOCH: 25/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:15:38) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0021474361419677734 [0m(+2.6464462280273438e-05) | |
| | > avg_loss_disc:[91m 2.44284725189209 [0m(+0.03238105773925781) | |
| | > avg_loss_disc_real_0:[91m 0.11237086355686188 [0m(+0.026415228843688965) | |
| | > avg_loss_disc_real_1:[91m 0.1936785876750946 [0m(+0.021366983652114868) | |
| | > avg_loss_disc_real_2:[92m 0.17568925023078918 [0m(-0.08402162790298462) | |
| | > avg_loss_disc_real_3:[92m 0.1501607447862625 [0m(-0.1047665923833847) | |
| | > avg_loss_disc_real_4:[91m 0.254810631275177 [0m(+0.05879028141498566) | |
| | > avg_loss_disc_real_5:[92m 0.18697001039981842 [0m(-0.07841582596302032) | |
| | > avg_loss_0:[91m 2.44284725189209 [0m(+0.03238105773925781) | |
| | > avg_loss_gen:[92m 1.9306268692016602 [0m(-0.5037117004394531) | |
| | > avg_loss_kl:[92m 2.0787177085876465 [0m(-0.11428570747375488) | |
| | > avg_loss_feat:[92m 6.487472057342529 [0m(-2.0779128074645996) | |
| | > avg_loss_mel:[92m 17.353788375854492 [0m(-1.6597175598144531) | |
| | > avg_loss_duration:[91m 1.5673844814300537 [0m(+0.0312192440032959) | |
| | > avg_loss_1:[92m 29.41798973083496 [0m(-4.324411392211914) | |
| [4m[1m > EPOCH: 26/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:16:03) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018668174743652344 [0m(-0.00028061866760253906) | |
| | > avg_loss_disc:[91m 2.4484174251556396 [0m(+0.005570173263549805) | |
| | > avg_loss_disc_real_0:[91m 0.2349560707807541 [0m(+0.12258520722389221) | |
| | > avg_loss_disc_real_1:[91m 0.19571739435195923 [0m(+0.002038806676864624) | |
| | > avg_loss_disc_real_2:[91m 0.1958414912223816 [0m(+0.020152240991592407) | |
| | > avg_loss_disc_real_3:[91m 0.24930566549301147 [0m(+0.09914492070674896) | |
| | > avg_loss_disc_real_4:[92m 0.24968351423740387 [0m(-0.005127117037773132) | |
| | > avg_loss_disc_real_5:[91m 0.2158995419740677 [0m(+0.028929531574249268) | |
| | > avg_loss_0:[91m 2.4484174251556396 [0m(+0.005570173263549805) | |
| | > avg_loss_gen:[91m 2.308260917663574 [0m(+0.37763404846191406) | |
| | > avg_loss_kl:[91m 2.1599037647247314 [0m(+0.08118605613708496) | |
| | > avg_loss_feat:[91m 6.9100341796875 [0m(+0.4225621223449707) | |
| | > avg_loss_mel:[92m 16.561206817626953 [0m(-0.7925815582275391) | |
| | > avg_loss_duration:[92m 1.5258960723876953 [0m(-0.0414884090423584) | |
| | > avg_loss_1:[91m 29.465303421020508 [0m(+0.047313690185546875) | |
| [4m[1m > EPOCH: 27/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:16:31) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0028181076049804688 [0m(+0.0009512901306152344) | |
| | > avg_loss_disc:[92m 2.3155648708343506 [0m(-0.13285255432128906) | |
| | > avg_loss_disc_real_0:[92m 0.14989317953586578 [0m(-0.0850628912448883) | |
| | > avg_loss_disc_real_1:[91m 0.23751142621040344 [0m(+0.041794031858444214) | |
| | > avg_loss_disc_real_2:[91m 0.23824138939380646 [0m(+0.042399898171424866) | |
| | > avg_loss_disc_real_3:[91m 0.25855839252471924 [0m(+0.009252727031707764) | |
| | > avg_loss_disc_real_4:[92m 0.23557884991168976 [0m(-0.014104664325714111) | |
| | > avg_loss_disc_real_5:[92m 0.18954725563526154 [0m(-0.026352286338806152) | |
| | > avg_loss_0:[92m 2.3155648708343506 [0m(-0.13285255432128906) | |
| | > avg_loss_gen:[91m 2.498218059539795 [0m(+0.1899571418762207) | |
| | > avg_loss_kl:[92m 2.103855609893799 [0m(-0.05604815483093262) | |
| | > avg_loss_feat:[91m 7.433771133422852 [0m(+0.5237369537353516) | |
| | > avg_loss_mel:[91m 18.378904342651367 [0m(+1.817697525024414) | |
| | > avg_loss_duration:[91m 1.541215419769287 [0m(+0.015319347381591797) | |
| | > avg_loss_1:[91m 31.955965042114258 [0m(+2.49066162109375) | |
| [4m[1m > EPOCH: 28/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:16:59) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0021948814392089844 [0m(-0.0006232261657714844) | |
| | > avg_loss_disc:[91m 2.719372272491455 [0m(+0.4038074016571045) | |
| | > avg_loss_disc_real_0:[91m 0.211879700422287 [0m(+0.061986520886421204) | |
| | > avg_loss_disc_real_1:[91m 0.27071627974510193 [0m(+0.033204853534698486) | |
| | > avg_loss_disc_real_2:[91m 0.28000608086586 [0m(+0.04176469147205353) | |
| | > avg_loss_disc_real_3:[91m 0.3032557964324951 [0m(+0.04469740390777588) | |
| | > avg_loss_disc_real_4:[91m 0.2568355202674866 [0m(+0.021256670355796814) | |
| | > avg_loss_disc_real_5:[91m 0.25513720512390137 [0m(+0.06558994948863983) | |
| | > avg_loss_0:[91m 2.719372272491455 [0m(+0.4038074016571045) | |
| | > avg_loss_gen:[92m 2.1684322357177734 [0m(-0.3297858238220215) | |
| | > avg_loss_kl:[92m 1.8129353523254395 [0m(-0.2909202575683594) | |
| | > avg_loss_feat:[92m 5.296431541442871 [0m(-2.1373395919799805) | |
| | > avg_loss_mel:[92m 16.067489624023438 [0m(-2.3114147186279297) | |
| | > avg_loss_duration:[92m 1.5332417488098145 [0m(-0.007973670959472656) | |
| | > avg_loss_1:[92m 26.878530502319336 [0m(-5.077434539794922) | |
| [4m[1m > EPOCH: 29/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:17:29) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018680095672607422 [0m(-0.0003268718719482422) | |
| | > avg_loss_disc:[91m 2.799192428588867 [0m(+0.07982015609741211) | |
| | > avg_loss_disc_real_0:[92m 0.18425635993480682 [0m(-0.027623340487480164) | |
| | > avg_loss_disc_real_1:[92m 0.2576143741607666 [0m(-0.013101905584335327) | |
| | > avg_loss_disc_real_2:[92m 0.24912326037883759 [0m(-0.0308828204870224) | |
| | > avg_loss_disc_real_3:[92m 0.26739048957824707 [0m(-0.03586530685424805) | |
| | > avg_loss_disc_real_4:[92m 0.20493660867214203 [0m(-0.05189891159534454) | |
| | > avg_loss_disc_real_5:[92m 0.17635811865329742 [0m(-0.07877908647060394) | |
| | > avg_loss_0:[91m 2.799192428588867 [0m(+0.07982015609741211) | |
| | > avg_loss_gen:[92m 1.9312785863876343 [0m(-0.23715364933013916) | |
| | > avg_loss_kl:[91m 1.9877099990844727 [0m(+0.1747746467590332) | |
| | > avg_loss_feat:[91m 6.73224401473999 [0m(+1.4358124732971191) | |
| | > avg_loss_mel:[92m 13.986883163452148 [0m(-2.080606460571289) | |
| | > avg_loss_duration:[92m 1.5008467435836792 [0m(-0.032395005226135254) | |
| | > avg_loss_1:[92m 26.138961791992188 [0m(-0.7395687103271484) | |
| [4m[1m > EPOCH: 30/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:17:56) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018367767333984375 [0m(-3.123283386230469e-05) | |
| | > avg_loss_disc:[92m 2.38435697555542 [0m(-0.41483545303344727) | |
| | > avg_loss_disc_real_0:[92m 0.12377185374498367 [0m(-0.06048450618982315) | |
| | > avg_loss_disc_real_1:[92m 0.18688198924064636 [0m(-0.07073238492012024) | |
| | > avg_loss_disc_real_2:[92m 0.23647478222846985 [0m(-0.012648478150367737) | |
| | > avg_loss_disc_real_3:[92m 0.18817488849163055 [0m(-0.07921560108661652) | |
| | > avg_loss_disc_real_4:[92m 0.17610274255275726 [0m(-0.028833866119384766) | |
| | > avg_loss_disc_real_5:[92m 0.09066959470510483 [0m(-0.0856885239481926) | |
| | > avg_loss_0:[92m 2.38435697555542 [0m(-0.41483545303344727) | |
| | > avg_loss_gen:[91m 2.238271713256836 [0m(+0.30699312686920166) | |
| | > avg_loss_kl:[92m 1.7533683776855469 [0m(-0.23434162139892578) | |
| | > avg_loss_feat:[91m 9.994836807250977 [0m(+3.2625927925109863) | |
| | > avg_loss_mel:[91m 19.69933319091797 [0m(+5.71245002746582) | |
| | > avg_loss_duration:[91m 1.5312981605529785 [0m(+0.030451416969299316) | |
| | > avg_loss_1:[91m 35.21710968017578 [0m(+9.078147888183594) | |
| [4m[1m > EPOCH: 31/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:18:22) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0018820762634277344 [0m(+4.5299530029296875e-05) | |
| | > avg_loss_disc:[91m 2.741990566253662 [0m(+0.3576335906982422) | |
| | > avg_loss_disc_real_0:[91m 0.1434936225414276 [0m(+0.01972176879644394) | |
| | > avg_loss_disc_real_1:[92m 0.1855868548154831 [0m(-0.001295134425163269) | |
| | > avg_loss_disc_real_2:[91m 0.24143913388252258 [0m(+0.004964351654052734) | |
| | > avg_loss_disc_real_3:[91m 0.20179814100265503 [0m(+0.013623252511024475) | |
| | > avg_loss_disc_real_4:[92m 0.17437708377838135 [0m(-0.0017256587743759155) | |
| | > avg_loss_disc_real_5:[91m 0.2018098384141922 [0m(+0.11114024370908737) | |
| | > avg_loss_0:[91m 2.741990566253662 [0m(+0.3576335906982422) | |
| | > avg_loss_gen:[92m 1.7299083471298218 [0m(-0.5083633661270142) | |
| | > avg_loss_kl:[91m 1.9740378856658936 [0m(+0.22066950798034668) | |
| | > avg_loss_feat:[92m 7.29171895980835 [0m(-2.703117847442627) | |
| | > avg_loss_mel:[92m 15.08013916015625 [0m(-4.619194030761719) | |
| | > avg_loss_duration:[92m 1.5062572956085205 [0m(-0.025040864944458008) | |
| | > avg_loss_1:[92m 27.582061767578125 [0m(-7.635047912597656) | |
| [4m[1m > EPOCH: 32/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:18:50) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.002040386199951172 [0m(+0.0001583099365234375) | |
| | > avg_loss_disc:[92m 2.652589797973633 [0m(-0.0894007682800293) | |
| | > avg_loss_disc_real_0:[92m 0.13831813633441925 [0m(-0.005175486207008362) | |
| | > avg_loss_disc_real_1:[91m 0.2390671968460083 [0m(+0.05348034203052521) | |
| | > avg_loss_disc_real_2:[91m 0.3314564526081085 [0m(+0.09001731872558594) | |
| | > avg_loss_disc_real_3:[91m 0.3158210217952728 [0m(+0.1140228807926178) | |
| | > avg_loss_disc_real_4:[91m 0.21650941669940948 [0m(+0.04213233292102814) | |
| | > avg_loss_disc_real_5:[92m 0.15263138711452484 [0m(-0.04917845129966736) | |
| | > avg_loss_0:[92m 2.652589797973633 [0m(-0.0894007682800293) | |
| | > avg_loss_gen:[91m 2.1515212059020996 [0m(+0.42161285877227783) | |
| | > avg_loss_kl:[92m 1.8530592918395996 [0m(-0.12097859382629395) | |
| | > avg_loss_feat:[91m 8.823948860168457 [0m(+1.5322299003601074) | |
| | > avg_loss_mel:[91m 19.046369552612305 [0m(+3.9662303924560547) | |
| | > avg_loss_duration:[91m 1.550673007965088 [0m(+0.04441571235656738) | |
| | > avg_loss_1:[91m 33.42557144165039 [0m(+5.843509674072266) | |
| [4m[1m > EPOCH: 33/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:19:18) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[92m 0.0018901824951171875 [0m(-0.00015020370483398438) | |
| | > avg_loss_disc:[92m 2.5330142974853516 [0m(-0.11957550048828125) | |
| | > avg_loss_disc_real_0:[91m 0.1612425148487091 [0m(+0.022924378514289856) | |
| | > avg_loss_disc_real_1:[92m 0.15939201414585114 [0m(-0.07967518270015717) | |
| | > avg_loss_disc_real_2:[92m 0.27959954738616943 [0m(-0.05185690522193909) | |
| | > avg_loss_disc_real_3:[92m 0.22299066185951233 [0m(-0.0928303599357605) | |
| | > avg_loss_disc_real_4:[91m 0.2748161256313324 [0m(+0.05830670893192291) | |
| | > avg_loss_disc_real_5:[91m 0.20549322664737701 [0m(+0.05286183953285217) | |
| | > avg_loss_0:[92m 2.5330142974853516 [0m(-0.11957550048828125) | |
| | > avg_loss_gen:[92m 2.130063533782959 [0m(-0.021457672119140625) | |
| | > avg_loss_kl:[92m 1.8332613706588745 [0m(-0.019797921180725098) | |
| | > avg_loss_feat:[92m 6.8745574951171875 [0m(-1.9493913650512695) | |
| | > avg_loss_mel:[92m 18.154199600219727 [0m(-0.8921699523925781) | |
| | > avg_loss_duration:[91m 1.5514920949935913 [0m(+0.000819087028503418) | |
| | > avg_loss_1:[92m 30.543575286865234 [0m(-2.8819961547851562) | |
| [4m[1m > EPOCH: 34/35[0m | |
| --> /home/azureuser/BanglaTTS/nctb-vits-single-male-9/vits-ft-nctb-March-31-2024_10+05AM-0000000 | |
| [1m > TRAINING (2024-03-31 10:19:48) [0m | |
| [1m > EVALUATION [0m | |
| [1m--> EVAL PERFORMANCE[0m | |
| | > avg_loader_time:[91m 0.0021767616271972656 [0m(+0.0002865791320800781) | |
| | > avg_loss_disc:[92m 2.4458301067352295 [0m(-0.08718419075012207) | |
| | > avg_loss_disc_real_0:[91m 0.18880796432495117 [0m(+0.027565449476242065) | |
| | > avg_loss_disc_real_1:[92m 0.15713262557983398 [0m(-0.002259388566017151) | |
| | > avg_loss_disc_real_2:[92m 0.2482978254556656 [0m(-0.031301721930503845) | |
| | > avg_loss_disc_real_3:[92m 0.21974673867225647 [0m(-0.0032439231872558594) | |
| | > avg_loss_disc_real_4:[92m 0.2329111099243164 [0m(-0.04190501570701599) | |
| | > avg_loss_disc_real_5:[91m 0.2461576908826828 [0m(+0.040664464235305786) | |
| | > avg_loss_0:[92m 2.4458301067352295 [0m(-0.08718419075012207) | |
| | > avg_loss_gen:[91m 2.317957878112793 [0m(+0.18789434432983398) | |
| | > avg_loss_kl:[91m 2.012809991836548 [0m(+0.17954862117767334) | |
| | > avg_loss_feat:[92m 6.7331624031066895 [0m(-0.14139509201049805) | |
| | > avg_loss_mel:[92m 16.866455078125 [0m(-1.2877445220947266) | |
| | > avg_loss_duration:[92m 1.5175280570983887 [0m(-0.03396403789520264) | |
| | > avg_loss_1:[92m 29.447914123535156 [0m(-1.0956611633300781) | |