from dataclasses import dataclass from typing import Optional, Tuple import torch from torch import Tensor, nn from torchaudio.models import Tacotron2 from transformers import PretrainedConfig, PreTrainedModel from transformers.utils import ModelOutput __version__ = "0.1.0" @dataclass class Tacotron2Output(ModelOutput): """ mel_outputs_postnet The predicted mel spectrogram with shape `(n_batch, n_mels, max of mel_specgram_lengths)`. mel_specgram_lengths The length of the predicted mel spectrogram with shape `(n_batch, )`. alignments Sequence of attention weights from the decoder with shape `(n_batch, max of mel_specgram_lengths, max of lengths)`. """ mel_outputs_postnet: Tensor = None mel_specgram_lengths: Tensor = None alignments: Tensor = None @dataclass class Tacotron2ForPreTrainingOutput(ModelOutput): """ mel_specgram Mel spectrogram before Postnet with shape `(n_batch, n_mels, max of mel_specgram_lengths)`. mel_specgram_postnet Mel spectrogram after Postnet with shape `(n_batch, n_mels, max of mel_specgram_lengths)`. gate_outputs The output for stop token at each time step with shape `(n_batch, max of mel_specgram_lengths)`. alignments Sequence of attention weights from the decoder with shape `(n_batch, max of mel_specgram_lengths, max of token_lengths)`. """ mel_specgram: Tensor = None mel_specgram_postnet: Tensor = None gate_outputs: Tensor = None alignments: Tensor = None loss: Optional[Tensor] = None mel_loss: Optional[Tensor] = None mel_postnet_loss: Optional[Tensor] = None gate_loss: Optional[Tensor] = None class Tacotron2Config(PretrainedConfig): def __init__( self, mask_padding: bool = False, n_mels: int = 80, n_symbol: int = 392, n_frames_per_step: int = 1, symbol_embedding_dim: int = 512, encoder_embedding_dim: int = 512, encoder_n_convolution: int = 3, encoder_kernel_size: int = 5, decoder_rnn_dim: int = 1024, decoder_max_step: int = 2000, decoder_dropout: float = 0.1, decoder_early_stopping: bool = True, attention_rnn_dim: int = 1024, attention_hidden_dim: int = 128, attention_location_n_filter: int = 32, attention_location_kernel_size: int = 31, attention_dropout: float = 0.1, prenet_dim: int = 256, postnet_n_convolution: int = 5, postnet_kernel_size: int = 5, postnet_embedding_dim: int = 512, gate_threshold: float = 0.5, **kwargs, ): # https://pytorch.org/audio/stable/generated/torchaudio.models.Tacotron2.html#torchaudio.models.Tacotron2 # noqa if n_frames_per_step != 1: raise ValueError( f"n_frames_per_step: only 1 is supported, got {n_frames_per_step}" ) self.mask_padding = mask_padding self.n_mels = n_mels self.n_symbol = n_symbol self.n_frames_per_step = n_frames_per_step self.symbol_embedding_dim = symbol_embedding_dim self.encoder_embedding_dim = encoder_embedding_dim self.encoder_n_convolution = encoder_n_convolution self.encoder_kernel_size = encoder_kernel_size self.decoder_rnn_dim = decoder_rnn_dim self.decoder_max_step = decoder_max_step self.decoder_dropout = decoder_dropout self.decoder_early_stopping = decoder_early_stopping self.attention_rnn_dim = attention_rnn_dim self.attention_hidden_dim = attention_hidden_dim self.attention_location_n_filter = attention_location_n_filter self.attention_location_kernel_size = attention_location_kernel_size self.attention_dropout = attention_dropout self.prenet_dim = prenet_dim self.postnet_n_convolution = postnet_n_convolution self.postnet_kernel_size = postnet_kernel_size self.postnet_embedding_dim = postnet_embedding_dim self.gate_threshold = gate_threshold super().__init__(**kwargs) class Tacotron2PreTrainedModel(PreTrainedModel): config_class = Tacotron2Config base_model_prefix = "tacotron2" main_input_name = "input_ids" class Tacotron2Model(Tacotron2PreTrainedModel): def __init__(self, config: Tacotron2Config): super().__init__(config) self.tacotron2 = Tacotron2( mask_padding=config.mask_padding, n_mels=config.n_mels, n_symbol=config.n_symbol, n_frames_per_step=config.n_frames_per_step, symbol_embedding_dim=config.symbol_embedding_dim, encoder_embedding_dim=config.encoder_embedding_dim, encoder_n_convolution=config.encoder_n_convolution, encoder_kernel_size=config.encoder_kernel_size, decoder_rnn_dim=config.decoder_rnn_dim, decoder_max_step=config.decoder_max_step, decoder_dropout=config.decoder_dropout, decoder_early_stopping=config.decoder_early_stopping, attention_rnn_dim=config.attention_rnn_dim, attention_hidden_dim=config.attention_hidden_dim, attention_location_n_filter=config.attention_location_n_filter, attention_location_kernel_size=config.attention_location_kernel_size, attention_dropout=config.attention_dropout, prenet_dim=config.prenet_dim, postnet_n_convolution=config.postnet_n_convolution, postnet_kernel_size=config.postnet_kernel_size, postnet_embedding_dim=config.postnet_embedding_dim, gate_threshold=config.gate_threshold, ) def forward( self, input_ids: Tensor, length: Optional[Tensor] = None, return_dict: Optional[bool] = None, ): r""" Using Tacotron2 for inference. The input is a batch of encoded sentences (``tokens``) and its corresponding lengths (``lengths``). The output is the generated mel spectrograms, its corresponding lengths, and the attention weights from the decoder. The input `tokens` should be padded with zeros to length max of ``lengths``. Args: tokens (Tensor): The input tokens to Tacotron2 with shape `(n_batch, max of lengths)`. lengths (Tensor or None, optional): The valid length of each sample in ``tokens`` with shape `(n_batch, )`. If ``None``, it is assumed that the all the tokens are valid. Default: ``None`` Returns: (Tensor, Tensor, Tensor): Tensor The predicted mel spectrogram with shape `(n_batch, n_mels, max of mel_specgram_lengths)`. Tensor The length of the predicted mel spectrogram with shape `(n_batch, )`. Tensor Sequence of attention weights from the decoder with shape `(n_batch, max of mel_specgram_lengths, max of lengths)`. """ return_dict = ( return_dict if return_dict is not None else self.config.use_return_dict ) outputs = self.tacotron2.infer(tokens=input_ids, lengths=length) if not return_dict: return outputs return Tacotron2Output( mel_outputs_postnet=outputs[0], mel_specgram_lengths=outputs[1], alignments=outputs[2], ) class Tacotron2Loss(nn.Module): """Tacotron2 loss function modified from: https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/SpeechSynthesis/Tacotron2/tacotron2/loss_function.py # noqa """ def __init__(self): super().__init__() self.mse_loss = nn.MSELoss(reduction="mean") self.bce_loss = nn.BCEWithLogitsLoss(reduction="mean") def forward( self, model_outputs: Tuple[Tensor, Tensor, Tensor], targets: Tuple[Tensor, Tensor], ) -> Tuple[Tensor, Tensor, Tensor]: r"""Pass the input through the Tacotron2 loss. The original implementation was introduced in *Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions* [:footcite:`shen2018natural`]. Args: model_outputs (tuple of three Tensors): The outputs of the Tacotron2. These outputs should include three items: (1) the predicted mel spectrogram before the postnet (``mel_specgram``) with shape (batch, mel, time). (2) predicted mel spectrogram after the postnet (``mel_specgram_postnet``) # noqa with shape (batch, mel, time), and (3) the stop token prediction (``gate_out``) with shape (batch, ). targets (tuple of two Tensors): The ground truth mel spectrogram (batch, mel, time) and stop token with shape (batch, ). Returns: mel_loss (Tensor): The mean MSE of the mel_specgram and ground truth mel spectrogram # noqa with shape ``torch.Size([])``. mel_postnet_loss (Tensor): The mean MSE of the mel_specgram_postnet and ground truth mel spectrogram with shape ``torch.Size([])``. gate_loss (Tensor): The mean binary cross entropy loss of the prediction on the stop token with shape ``torch.Size([])``. """ mel_target, gate_target = targets[0], targets[1] gate_target = gate_target.view(-1, 1) mel_specgram, mel_specgram_postnet, gate_out = model_outputs gate_out = gate_out.view(-1, 1) mel_loss = self.mse_loss(mel_specgram, mel_target) mel_postnet_loss = self.mse_loss(mel_specgram_postnet, mel_target) gate_loss = self.bce_loss(gate_out, gate_target) return mel_loss, mel_postnet_loss, gate_loss class Tacotron2ForPreTraining(Tacotron2PreTrainedModel): def __init__(self, config: Tacotron2Config): super().__init__(config) self.tacotron2 = Tacotron2( mask_padding=config.mask_padding, n_mels=config.n_mels, n_symbol=config.n_symbol, n_frames_per_step=config.n_frames_per_step, symbol_embedding_dim=config.symbol_embedding_dim, encoder_embedding_dim=config.encoder_embedding_dim, encoder_n_convolution=config.encoder_n_convolution, encoder_kernel_size=config.encoder_kernel_size, decoder_rnn_dim=config.decoder_rnn_dim, decoder_max_step=config.decoder_max_step, decoder_dropout=config.decoder_dropout, decoder_early_stopping=config.decoder_early_stopping, attention_rnn_dim=config.attention_rnn_dim, attention_hidden_dim=config.attention_hidden_dim, attention_location_n_filter=config.attention_location_n_filter, attention_location_kernel_size=config.attention_location_kernel_size, attention_dropout=config.attention_dropout, prenet_dim=config.prenet_dim, postnet_n_convolution=config.postnet_n_convolution, postnet_kernel_size=config.postnet_kernel_size, postnet_embedding_dim=config.postnet_embedding_dim, gate_threshold=config.gate_threshold, ) self.loss_fct = Tacotron2Loss() def sync_batchnorm(self): self.tacotron2 = nn.SyncBatchNorm.convert_sync_batchnorm(self.tacotron2) def forward( self, input_ids: Tensor, length: Tensor, mel_specgram: Tensor, mel_specgram_length: Tensor, gate_padded: Optional[Tensor] = None, return_dict: Optional[bool] = None, ): return_dict = ( return_dict if return_dict is not None else self.config.use_return_dict ) outputs = self.tacotron2( tokens=input_ids, token_lengths=length, mel_specgram=mel_specgram, mel_specgram_lengths=mel_specgram_length, ) loss = mel_loss = mel_postnet_loss = gate_loss = None if gate_padded is not None: targets = (mel_specgram, gate_padded) targets[0].requires_grad = False targets[1].requires_grad = False mel_loss, mel_postnet_loss, gate_loss = self.loss_fct(outputs[:3], targets) loss = mel_loss + mel_postnet_loss + gate_loss if not return_dict: if loss is not None: return outputs + (loss, mel_loss, mel_postnet_loss, gate_loss) return outputs return Tacotron2ForPreTrainingOutput( mel_specgram=outputs[0], mel_specgram_postnet=outputs[1], gate_outputs=outputs[2], alignments=outputs[3], loss=loss, mel_loss=mel_loss, mel_postnet_loss=mel_postnet_loss, gate_loss=gate_loss, )