vocalist contentvec and whisper as content features

.gitignore

@@ -0,0 +1 @@
+result

vocalist_l1_contentvec+whisper/args.json

@@ -0,0 +1,256 @@
+{
+    "base_config": "config/diffusion.json",
+    "dataset": [
+        "vocalist_l1",
+    ],
+    "exp_name": "vocalist_l1_contentvec+whisper",
+    "inference": {
+        "diffusion": {
+            "scheduler": "pndm",
+            "scheduler_settings": {
+                "num_inference_timesteps": 1000,
+            },
+        },
+    },
+    "model": {
+        "condition_encoder": {
+            "content_encoder_dim": 384,
+            "contentvec_dim": 256,
+            "f0_max": 1100,
+            "f0_min": 50,
+            "input_loudness_dim": 1,
+            "input_melody_dim": 1,
+            "merge_mode": "add",
+            "mert_dim": 256,
+            "n_bins_loudness": 256,
+            "n_bins_melody": 256,
+            "output_content_dim": 384,
+            "output_loudness_dim": 384,
+            "output_melody_dim": 384,
+            "output_singer_dim": 384,
+            "pitch_max": 1100,
+            "pitch_min": 50,
+            "singer_table_size": 512,
+            "use_conformer_for_content_features": false,
+            "use_contentvec": true,
+            "use_log_f0": true,
+            "use_log_loudness": true,
+            "use_mert": false,
+            "use_singer_encoder": true,
+            "use_spkid": true,
+            "use_wenet": false,
+            "use_whisper": true,
+            "wenet_dim": 512,
+            "whisper_dim": 1024,
+        },
+        "diffusion": {
+            "bidilconv": {
+                "base_channel": 384,
+                "conditioner_size": 384,
+                "conv_kernel_size": 3,
+                "dilation_cycle_length": 4,
+                "n_res_block": 20,
+            },
+            "model_type": "bidilconv",
+            "scheduler": "ddpm",
+            "scheduler_settings": {
+                "beta_end": 0.02,
+                "beta_schedule": "linear",
+                "beta_start": 0.0001,
+                "num_train_timesteps": 1000,
+            },
+            "step_encoder": {
+                "activation": "SiLU",
+                "dim_hidden_layer": 512,
+                "dim_raw_embedding": 128,
+                "max_period": 10000,
+                "num_layer": 2,
+            },
+            "unet2d": {
+                "down_block_types": [
+                    "CrossAttnDownBlock2D",
+                    "CrossAttnDownBlock2D",
+                    "CrossAttnDownBlock2D",
+                    "DownBlock2D",
+                ],
+                "in_channels": 1,
+                "mid_block_type": "UNetMidBlock2DCrossAttn",
+                "only_cross_attention": false,
+                "out_channels": 1,
+                "up_block_types": [
+                    "UpBlock2D",
+                    "CrossAttnUpBlock2D",
+                    "CrossAttnUpBlock2D",
+                    "CrossAttnUpBlock2D",
+                ],
+            },
+        },
+    },
+    "model_type": "DiffWaveNetSVC",
+    "preprocess": {
+        "audio_dir": "audios",
+        "bits": 8,
+        "content_feature_batch_size": 16,
+        "contentvec_batch_size": 1,
+        "contentvec_dir": "contentvec",
+        "contentvec_file": "pretrained/contentvec/checkpoint_best_legacy_500.pt",
+        "contentvec_frameshift": 0.02,
+        "contentvec_sample_rate": 16000,
+        "dur_dir": "durs",
+        "duration_dir": "duration",
+        "emo2id": "emo2id.json",
+        "energy_dir": "energys",
+        "extract_audio": false,
+        "extract_contentvec_feature": true,
+        "extract_energy": true,
+        "extract_label": false,
+        "extract_mcep": false,
+        "extract_mel": true,
+        "extract_mert_feature": false,
+        "extract_pitch": true,
+        "extract_uv": true,
+        "extract_wenet_feature": false,
+        "extract_whisper_feature": true,
+        "f0_max": 1100,
+        "f0_min": 50,
+        "file_lst": "file.lst",
+        "fmax": 12000,
+        "fmin": 0,
+        "hop_size": 256,
+        "is_label": true,
+        "is_mu_law": true,
+        "lab_dir": "labs",
+        "label_dir": "labels",
+        "mcep_dir": "mcep",
+        "mel_dir": "mels",
+        "mel_min_max_norm": true,
+        "mel_min_max_stats_dir": "mel_min_max_stats",
+        "mert_dir": "mert",
+        "mert_feature_layer": -1,
+        "mert_frameshit": 0.01333,
+        "mert_hop_size": 320,
+        "mert_model": "m-a-p/MERT-v1-330M",
+        "min_level_db": -115,
+        "mu_law_norm": false,
+        "n_fft": 1024,
+        "n_mel": 100,
+        "num_silent_frames": 8,
+        "num_workers": 8,
+        "phone_seq_file": "phone_seq_file",
+        "pin_memory": true,
+        "pitch_bin": 256,
+        "pitch_dir": "pitches",
+        "pitch_extractor": "parselmouth",
+        "pitch_max": 1100.0,
+        "pitch_min": 50.0,
+        "processed_dir": "ckpts/svc/vocalist_l1_contentvec+whisper/data",
+        "ref_level_db": 20,
+        "sample_rate": 24000,
+        "spk2id": "singers.json",
+        "train_file": "train.json",
+        "trim_fft_size": 512,
+        "trim_hop_size": 128,
+        "trim_silence": false,
+        "trim_top_db": 30,
+        "trimmed_wav_dir": "trimmed_wavs",
+        "use_audio": false,
+        "use_contentvec": true,
+        "use_dur": false,
+        "use_emoid": false,
+        "use_frame_duration": false,
+        "use_frame_energy": true,
+        "use_frame_pitch": true,
+        "use_lab": false,
+        "use_label": false,
+        "use_log_scale_energy": false,
+        "use_log_scale_pitch": false,
+        "use_mel": true,
+        "use_mert": false,
+        "use_min_max_norm_mel": true,
+        "use_one_hot": false,
+        "use_phn_seq": false,
+        "use_phone_duration": false,
+        "use_phone_energy": false,
+        "use_phone_pitch": false,
+        "use_spkid": true,
+        "use_uv": true,
+        "use_wav": false,
+        "use_wenet": false,
+        "use_whisper": true,
+        "utt2emo": "utt2emo",
+        "utt2spk": "utt2singer",
+        "uv_dir": "uvs",
+        "valid_file": "test.json",
+        "wav_dir": "wavs",
+        "wenet_batch_size": 1,
+        "wenet_config": "pretrained/wenet/20220506_u2pp_conformer_exp/train.yaml",
+        "wenet_dir": "wenet",
+        "wenet_downsample_rate": 4,
+        "wenet_frameshift": 0.01,
+        "wenet_model_path": "pretrained/wenet/20220506_u2pp_conformer_exp/final.pt",
+        "wenet_sample_rate": 16000,
+        "whisper_batch_size": 30,
+        "whisper_dir": "whisper",
+        "whisper_downsample_rate": 2,
+        "whisper_frameshift": 0.01,
+        "whisper_model": "medium",
+        "whisper_model_path": "pretrained/whisper/medium.pt",
+        "win_size": 1024,
+    },
+    "supported_model_type": [
+        "Fastspeech2",
+        "DiffSVC",
+        "Transformer",
+        "EDM",
+        "CD",
+    ],
+    "train": {
+        "adamw": {
+            "lr": 0.0004,
+        },
+        "batch_size": 32,
+        "dataloader": {
+            "num_worker": 8,
+            "pin_memory": true,
+        },
+        "ddp": true,
+        "epochs": 50000,
+        "gradient_accumulation_step": 1,
+        "keep_checkpoint_max": 5,
+        "keep_last": [
+            5,
+            -1,
+        ],
+        "max_epoch": -1,
+        "max_steps": 1000000,
+        "multi_speaker_training": false,
+        "optimizer": "AdamW",
+        "random_seed": 10086,
+        "reducelronplateau": {
+            "factor": 0.8,
+            "min_lr": 0.0001,
+            "patience": 10,
+        },
+        "run_eval": [
+            false,
+            true,
+        ],
+        "sampler": {
+            "drop_last": true,
+            "holistic_shuffle": false,
+        },
+        "save_checkpoint_stride": [
+            3,
+            10,
+        ],
+        "save_checkpoints_steps": 10000,
+        "save_summary_steps": 500,
+        "scheduler": "ReduceLROnPlateau",
+        "total_training_steps": 50000,
+        "tracker": [
+            "tensorboard",
+        ],
+        "valid_interval": 10000,
+    },
+    "use_custom_dataset": true,
+}

vocalist_l1_contentvec+whisper/checkpoint/epoch-6852_step-0678447_loss-1.946773/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:836af10b834c7aec9209eb19ce43559e6ef1e3a59bd6468e90cadbc9a18749ef
+size 249512389

vocalist_l1_contentvec+whisper/checkpoint/epoch-6852_step-0678447_loss-1.946773/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d54eed12bef331095fc367f196d07c5061d5cb72dd6fe0e1e4453b997bf1d68d
+size 124755137

vocalist_l1_contentvec+whisper/checkpoint/epoch-6852_step-0678447_loss-1.946773/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6798ddffadcd7d5405a77e667c674c474e4fef0cba817fdd300c7c985c1e82fe
+size 14599

vocalist_l1_contentvec+whisper/data/vocalist_l1/mel_min_max_stats/mel_max.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04131849378aa4f525a701909f743c303f8d56571682572b888046ead9f3e2ab
+size 528

vocalist_l1_contentvec+whisper/data/vocalist_l1/mel_min_max_stats/mel_min.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef4895ebef0e9949a6e623315bdc8a68490ba95d2f81b2be9f5146f904203016
+size 528

vocalist_l1_contentvec+whisper/data/vocalist_l1/pitches/statistics.json

@@ -0,0 +1,242 @@
+{
+    "vocalist_l1_Adele": {
+        "voiced_positions": {
+            "mean": 336.5038018286193,
+            "std": 100.2148774476881,
+            "median": 332.98363792619296,
+            "min": 59.99838412340723,
+            "max": 1099.849325287837
+        },
+        "total_positions": {
+            "mean": 231.79366581704338,
+            "std": 176.6042850107386,
+            "median": 273.2844263775394,
+            "min": 0.0,
+            "max": 1099.849325287837
+        }
+    },
+    "vocalist_l1_Beyonce": {
+        "voiced_positions": {
+            "mean": 357.5678927636881,
+            "std": 130.1132620135807,
+            "median": 318.2981879228934,
+            "min": 70.29719673914867,
+            "max": 1050.354470112099
+        },
+        "total_positions": {
+            "mean": 267.5248026267327,
+            "std": 191.71600807951046,
+            "median": 261.91981963774066,
+            "min": 0.0,
+            "max": 1050.354470112099
+        }
+    },
+    "vocalist_l1_BrunoMars": {
+        "voiced_positions": {
+            "mean": 330.92612740814315,
+            "std": 86.51034158515388,
+            "median": 324.65585832605217,
+            "min": 58.74277302450286,
+            "max": 999.2818302992808
+        },
+        "total_positions": {
+            "mean": 237.26076288057826,
+            "std": 166.09898203490803,
+            "median": 286.3097386522132,
+            "min": 0.0,
+            "max": 999.2818302992808
+        }
+    },
+    "vocalist_l1_JohnMayer": {
+        "voiced_positions": {
+            "mean": 218.3531239166661,
+            "std": 77.89887175223768,
+            "median": 200.19060542586652,
+            "min": 53.371912740674716,
+            "max": 1098.1986774161685
+        },
+        "total_positions": {
+            "mean": 112.95331907131244,
+            "std": 122.65534824070893,
+            "median": 124.71389285965317,
+            "min": 0.0,
+            "max": 1098.1986774161685
+        }
+    },
+    "vocalist_l1_MichaelJackson": {
+        "voiced_positions": {
+            "mean": 293.4663654519906,
+            "std": 89.02211325650234,
+            "median": 284.4323483619402,
+            "min": 61.14507754070825,
+            "max": 1096.4247902272325
+        },
+        "total_positions": {
+            "mean": 172.1013565770682,
+            "std": 159.79551912957191,
+            "median": 212.82938711725973,
+            "min": 0.0,
+            "max": 1096.4247902272325
+        }
+    },
+    "vocalist_l1_TaylorSwift": {
+        "voiced_positions": {
+            "mean": 302.5346928039029,
+            "std": 87.1724728626562,
+            "median": 286.91670244246586,
+            "min": 51.31173137207717,
+            "max": 1098.9374311806605
+        },
+        "total_positions": {
+            "mean": 169.90968097339214,
+            "std": 163.7133164876362,
+            "median": 220.90943653386546,
+            "min": 0.0,
+            "max": 1098.9374311806605
+        }
+    },
+    "vocalist_l1_张学友": {
+        "voiced_positions": {
+            "mean": 233.6845479691867,
+            "std": 66.47140810463938,
+            "median": 228.28695118043396,
+            "min": 51.65338480121057,
+            "max": 1094.4381927885959
+        },
+        "total_positions": {
+            "mean": 167.79543637603194,
+            "std": 119.28338415844308,
+            "median": 194.81504136428546,
+            "min": 0.0,
+            "max": 1094.4381927885959
+        }
+    },
+    "vocalist_l1_李健": {
+        "voiced_positions": {
+            "mean": 234.98401896504657,
+            "std": 71.3955175177514,
+            "median": 221.86415264367847,
+            "min": 54.070687769392585,
+            "max": 1096.3342286660531
+        },
+        "total_positions": {
+            "mean": 148.74760079412246,
+            "std": 126.70486473504008,
+            "median": 180.21374566147688,
+            "min": 0.0,
+            "max": 1096.3342286660531
+        }
+    },
+    "vocalist_l1_汪峰": {
+        "voiced_positions": {
+            "mean": 284.27752567207864,
+            "std": 78.51774150654873,
+            "median": 278.26186808969493,
+            "min": 54.30945929095861,
+            "max": 1053.6870553733015
+        },
+        "total_positions": {
+            "mean": 172.41584497486713,
+            "std": 151.74272125914902,
+            "median": 216.27534661524862,
+            "min": 0.0,
+            "max": 1053.6870553733015
+        }
+    },
+    "vocalist_l1_王菲": {
+        "voiced_positions": {
+            "mean": 339.1661679865587,
+            "std": 86.86768172635271,
+            "median": 327.4151031268507,
+            "min": 51.21299842481366,
+            "max": 1096.7044574066776
+        },
+        "total_positions": {
+            "mean": 217.726880186,
+            "std": 176.8748978138034,
+            "median": 277.8608050501477,
+            "min": 0.0,
+            "max": 1096.7044574066776
+        }
+    },
+    "vocalist_l1_石倚洁": {
+        "voiced_positions": {
+            "mean": 279.67710779262256,
+            "std": 87.82306577322389,
+            "median": 271.13024912248443,
+            "min": 59.604772357481075,
+            "max": 1098.0574674417153
+        },
+        "total_positions": {
+            "mean": 205.49634806008135,
+            "std": 144.6064344590865,
+            "median": 234.19454400899718,
+            "min": 0.0,
+            "max": 1098.0574674417153
+        }
+    },
+    "vocalist_l1_蔡琴": {
+        "voiced_positions": {
+            "mean": 258.9105806499278,
+            "std": 67.4079737418162,
+            "median": 250.29778287949176,
+            "min": 54.81875790199644,
+            "max": 930.3733192171918
+        },
+        "total_positions": {
+            "mean": 197.64675891035662,
+            "std": 124.80889987119957,
+            "median": 228.14775033720753,
+            "min": 0.0,
+            "max": 930.3733192171918
+        }
+    },
+    "vocalist_l1_那英": {
+        "voiced_positions": {
+            "mean": 358.98655838013195,
+            "std": 91.30591323348871,
+            "median": 346.95185476261275,
+            "min": 71.62879029165369,
+            "max": 1085.4349856526985
+        },
+        "total_positions": {
+            "mean": 243.83317702162077,
+            "std": 183.68660712060583,
+            "median": 294.9745603259994,
+            "min": 0.0,
+            "max": 1085.4349856526985
+        }
+    },
+    "vocalist_l1_陈奕迅": {
+        "voiced_positions": {
+            "mean": 222.0124146654594,
+            "std": 68.65002654904572,
+            "median": 218.9200565540147,
+            "min": 50.48503062529368,
+            "max": 1084.6336454006018
+        },
+        "total_positions": {
+            "mean": 154.2275169157727,
+            "std": 117.16740631313343,
+            "median": 176.89315636838086,
+            "min": 0.0,
+            "max": 1084.6336454006018
+        }
+    },
+    "vocalist_l1_陶喆": {
+        "voiced_positions": {
+            "mean": 242.58206762395713,
+            "std": 69.61805791083957,
+            "median": 227.5222796096177,
+            "min": 50.44809060945403,
+            "max": 1098.4942623171203
+        },
+        "total_positions": {
+            "mean": 171.59040988406485,
+            "std": 124.93911390018495,
+            "median": 204.4328861811408,
+            "min": 0.0,
+            "max": 1098.4942623171203
+        }
+    }
+}

vocalist_l1_contentvec+whisper/log/vocalist_l1_contentvec+whisper/events.out.tfevents.1696052302.mmnewyardnodesz63219.120.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7f490fd0c97876e24bfc44413365ded7ff5d22c1c79f0dac0b754f3b32df76f
+size 88

vocalist_l1_contentvec+whisper/log/vocalist_l1_contentvec+whisper/events.out.tfevents.1696052302.mmnewyardnodesz63219.120.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e01bcf2fa621ba563b70568c18fe0742d0f48cafae83a6e8beb0bb6d1f6d146d
+size 77413046

vocalist_l1_contentvec+whisper/singers.json

@@ -0,0 +1,17 @@
+{
+    "vocalist_l1_Adele": 0,
+    "vocalist_l1_Beyonce": 1,
+    "vocalist_l1_BrunoMars": 2,
+    "vocalist_l1_JohnMayer": 3,
+    "vocalist_l1_MichaelJackson": 4,
+    "vocalist_l1_TaylorSwift": 5,
+    "vocalist_l1_张学友": 6,
+    "vocalist_l1_李健": 7,
+    "vocalist_l1_汪峰": 8,
+    "vocalist_l1_王菲": 9,
+    "vocalist_l1_石倚洁": 10,
+    "vocalist_l1_蔡琴": 11,
+    "vocalist_l1_那英": 12,
+    "vocalist_l1_陈奕迅": 13,
+    "vocalist_l1_陶喆": 14
+}