Spaces:
Running
on
Zero
Running
on
Zero
File size: 3,931 Bytes
2ccf6b5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
'''
https://github.com/cantabile-kwok/VoiceFlow-TTS/blob/main/model/diffsinger.py#L51
This is the original implementation of the DiffSinger model.
It is a slightly modified WV which can be used for initial tests.
Will update this into original p-flow implementation later.
'''
import math
import torch.nn as nn
import torch
from torch.nn import Conv1d, Linear
import math
import torch.nn.functional as F
class Mish(nn.Module):
def forward(self, x):
return x * torch.tanh(F.softplus(x))
class SinusoidalPosEmb(nn.Module):
def __init__(self, dim):
super(SinusoidalPosEmb, self).__init__()
self.dim = dim
def forward(self, x, scale=1000):
if x.ndim < 1:
x = x.unsqueeze(0)
device = x.device
half_dim = self.dim // 2
emb = math.log(10000) / (half_dim - 1)
emb = torch.exp(torch.arange(half_dim, device=device).float() * -emb)
emb = scale * x.unsqueeze(1) * emb.unsqueeze(0)
emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
return emb
class ResidualBlock(nn.Module):
def __init__(self, encoder_hidden, residual_channels, dilation):
super().__init__()
self.dilated_conv = Conv1d(residual_channels, 2 * residual_channels, 3, padding=dilation, dilation=dilation)
self.diffusion_projection = Linear(residual_channels, residual_channels)
self.conditioner_projection = Conv1d(encoder_hidden, 2 * residual_channels, 1)
self.output_projection = Conv1d(residual_channels, 2 * residual_channels, 1)
def forward(self, x, conditioner, diffusion_step):
diffusion_step = self.diffusion_projection(diffusion_step).unsqueeze(-1)
conditioner = self.conditioner_projection(conditioner)
y = x + diffusion_step
y = self.dilated_conv(y) + conditioner
gate, filter = torch.chunk(y, 2, dim=1)
y = torch.sigmoid(gate) * torch.tanh(filter)
y = self.output_projection(y)
residual, skip = torch.chunk(y, 2, dim=1)
return (x + residual) / math.sqrt(2.0), skip
class DiffSingerNet(nn.Module):
def __init__(
self,
in_dims=80,
residual_channels=256,
encoder_hidden=80,
dilation_cycle_length=1,
residual_layers=20,
pe_scale=1000
):
super().__init__()
self.pe_scale = pe_scale
self.input_projection = Conv1d(in_dims, residual_channels, 1)
self.time_pos_emb = SinusoidalPosEmb(residual_channels)
dim = residual_channels
self.mlp = nn.Sequential(
nn.Linear(dim, dim * 4),
Mish(),
nn.Linear(dim * 4, dim)
)
self.residual_layers = nn.ModuleList([
ResidualBlock(encoder_hidden, residual_channels, 2 ** (i % dilation_cycle_length))
for i in range(residual_layers)
])
self.skip_projection = Conv1d(residual_channels, residual_channels, 1)
self.output_projection = Conv1d(residual_channels, in_dims, 1)
nn.init.zeros_(self.output_projection.weight)
def forward(self, spec, spec_mask, mu, t, *args, **kwargs):
"""
:param spec: [B, M, T]
:param t: [B, ]
:param mu: [B, M, T]
:return:
"""
# x = spec[:, 0]
x = spec
x = self.input_projection(x) # x [B, residual_channel, T]
x = F.relu(x)
t = self.time_pos_emb(t, scale=self.pe_scale)
t = self.mlp(t)
cond = mu
skip = []
for layer_id, layer in enumerate(self.residual_layers):
x, skip_connection = layer(x, cond, t)
skip.append(skip_connection)
x = torch.sum(torch.stack(skip), dim=0) / math.sqrt(len(self.residual_layers))
x = self.skip_projection(x)
x = F.relu(x)
x = self.output_projection(x) # [B, M, T]
return x * spec_mask |