Spaces:
Running
Running
| import hashlib | |
| import json | |
| import logging | |
| import os | |
| import time | |
| from pathlib import Path | |
| import io | |
| import librosa | |
| import maad | |
| import numpy as np | |
| from inference import slicer | |
| import parselmouth | |
| import soundfile | |
| import torch | |
| import torchaudio | |
| from hubert import hubert_model | |
| import utils | |
| from models import SynthesizerTrn | |
| logging.getLogger('numba').setLevel(logging.WARNING) | |
| logging.getLogger('matplotlib').setLevel(logging.WARNING) | |
| def resize2d_f0(x, target_len): | |
| source = np.array(x) | |
| source[source < 0.001] = np.nan | |
| target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)), | |
| source) | |
| res = np.nan_to_num(target) | |
| return res | |
| def get_f0(x, p_len,f0_up_key=0): | |
| time_step = 160 / 16000 * 1000 | |
| f0_min = 50 | |
| f0_max = 1100 | |
| f0_mel_min = 1127 * np.log(1 + f0_min / 700) | |
| f0_mel_max = 1127 * np.log(1 + f0_max / 700) | |
| f0 = parselmouth.Sound(x, 16000).to_pitch_ac( | |
| time_step=time_step / 1000, voicing_threshold=0.6, | |
| pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency'] | |
| pad_size=(p_len - len(f0) + 1) // 2 | |
| if(pad_size>0 or p_len - len(f0) - pad_size>0): | |
| f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant') | |
| f0 *= pow(2, f0_up_key / 12) | |
| f0_mel = 1127 * np.log(1 + f0 / 700) | |
| f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1 | |
| f0_mel[f0_mel <= 1] = 1 | |
| f0_mel[f0_mel > 255] = 255 | |
| f0_coarse = np.rint(f0_mel).astype(np.int) | |
| return f0_coarse, f0 | |
| def clean_pitch(input_pitch): | |
| num_nan = np.sum(input_pitch == 1) | |
| if num_nan / len(input_pitch) > 0.9: | |
| input_pitch[input_pitch != 1] = 1 | |
| return input_pitch | |
| def plt_pitch(input_pitch): | |
| input_pitch = input_pitch.astype(float) | |
| input_pitch[input_pitch == 1] = np.nan | |
| return input_pitch | |
| def f0_to_pitch(ff): | |
| f0_pitch = 69 + 12 * np.log2(ff / 440) | |
| return f0_pitch | |
| def fill_a_to_b(a, b): | |
| if len(a) < len(b): | |
| for _ in range(0, len(b) - len(a)): | |
| a.append(a[0]) | |
| def mkdir(paths: list): | |
| for path in paths: | |
| if not os.path.exists(path): | |
| os.mkdir(path) | |
| class VitsSvc(object): | |
| def __init__(self): | |
| self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| self.SVCVITS = None | |
| self.hps = None | |
| self.speakers = None | |
| self.hubert_soft = utils.get_hubert_model() | |
| def set_device(self, device): | |
| self.device = torch.device(device) | |
| self.hubert_soft.to(self.device) | |
| if self.SVCVITS != None: | |
| self.SVCVITS.to(self.device) | |
| def loadCheckpoint(self, path): | |
| self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json") | |
| self.SVCVITS = SynthesizerTrn( | |
| self.hps.data.filter_length // 2 + 1, | |
| self.hps.train.segment_size // self.hps.data.hop_length, | |
| **self.hps.model) | |
| _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None) | |
| _ = self.SVCVITS.eval().to(self.device) | |
| self.speakers = self.hps.spk | |
| def get_units(self, source, sr): | |
| source = source.unsqueeze(0).to(self.device) | |
| with torch.inference_mode(): | |
| units = self.hubert_soft.units(source) | |
| return units | |
| def get_unit_pitch(self, in_path, tran): | |
| source, sr = torchaudio.load(in_path) | |
| source = torchaudio.functional.resample(source, sr, 16000) | |
| if len(source.shape) == 2 and source.shape[1] >= 2: | |
| source = torch.mean(source, dim=0).unsqueeze(0) | |
| soft = self.get_units(source, sr).squeeze(0).cpu().numpy() | |
| f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran) | |
| return soft, f0 | |
| def infer(self, speaker_id, tran, raw_path): | |
| speaker_id = self.speakers[speaker_id] | |
| sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0) | |
| soft, pitch = self.get_unit_pitch(raw_path, tran) | |
| f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device) | |
| stn_tst = torch.FloatTensor(soft) | |
| with torch.no_grad(): | |
| x_tst = stn_tst.unsqueeze(0).to(self.device) | |
| x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2) | |
| audio = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0,0].data.float() | |
| return audio, audio.shape[-1] | |
| def inference(self,srcaudio,chara,tran,slice_db): | |
| sampling_rate, audio = srcaudio | |
| audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32) | |
| if len(audio.shape) > 1: | |
| audio = librosa.to_mono(audio.transpose(1, 0)) | |
| if sampling_rate != 16000: | |
| audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000) | |
| soundfile.write("tmpwav.wav", audio, 16000, format="wav") | |
| chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db) | |
| audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks) | |
| audio = [] | |
| for (slice_tag, data) in audio_data: | |
| length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate)) | |
| raw_path = io.BytesIO() | |
| soundfile.write(raw_path, data, audio_sr, format="wav") | |
| raw_path.seek(0) | |
| if slice_tag: | |
| _audio = np.zeros(length) | |
| else: | |
| out_audio, out_sr = self.infer(chara, tran, raw_path) | |
| _audio = out_audio.cpu().numpy() | |
| audio.extend(list(_audio)) | |
| audio = (np.array(audio) * 32768.0).astype('int16') | |
| return (self.hps.data.sampling_rate,audio) | |