# ############################################################################ # Model: E2E ASR with Transformer # Encoder: Conformer Encoder # Decoder: Transformer Decoder + (CTC/ATT joint) beamsearch + TransformerLM # Tokens: unigram # losses: CTC + KLdiv (Label Smoothing loss) # Training: KsponSpeech 965.2h # Based on the works of: Jianyuan Zhong, Titouan Parcollet 2021 # Authors: Dongwon Kim, Dongwoo Kim 2021 # ############################################################################ # Seed needs to be set at top of yaml, before objects with parameters are made # Feature parameters sample_rate: 16000 n_fft: 400 n_mels: 80 ####################### Model parameters ########################### # Transformer d_model: 256 nhead: 4 num_encoder_layers: 12 num_decoder_layers: 6 d_ffn: 2048 transformer_dropout: 0.0 activation: !name:torch.nn.GELU output_neurons: 5000 vocab_size: 5000 # Outputs blank_index: 0 label_smoothing: 0.1 pad_index: 0 bos_index: 1 eos_index: 2 unk_index: 0 # Decoding parameters min_decode_ratio: 0.0 max_decode_ratio: 1.0 valid_search_interval: 10 valid_beam_size: 10 test_beam_size: 60 lm_weight: 0.20 ctc_weight_decode: 0.40 ############################## models ################################ normalizer: !new:speechbrain.processing.features.InputNormalization norm_type: global CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd input_shape: (8, 10, 80) num_blocks: 2 num_layers_per_block: 1 out_channels: (64, 32) kernel_sizes: (3, 3) strides: (2, 2) residuals: (False, False) Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length input_size: 640 tgt_vocab: !ref <output_neurons> d_model: !ref <d_model> nhead: !ref <nhead> num_encoder_layers: !ref <num_encoder_layers> num_decoder_layers: !ref <num_decoder_layers> d_ffn: !ref <d_ffn> dropout: !ref <transformer_dropout> activation: !ref <activation> encoder_module: conformer attention_type: RelPosMHAXL normalize_before: True causal: False # NB: It has to match the pre-trained TransformerLM!! lm_model: !new:speechbrain.lobes.models.transformer.TransformerLM.TransformerLM # yamllint disable-line rule:line-length vocab: !ref <output_neurons> d_model: 768 nhead: 12 num_encoder_layers: 12 num_decoder_layers: 0 d_ffn: 3072 dropout: 0.0 activation: !name:torch.nn.GELU normalize_before: False tokenizer: !new:sentencepiece.SentencePieceProcessor ctc_lin: !new:speechbrain.nnet.linear.Linear input_size: !ref <d_model> n_neurons: !ref <output_neurons> seq_lin: !new:speechbrain.nnet.linear.Linear input_size: !ref <d_model> n_neurons: !ref <output_neurons> decoder: !new:speechbrain.decoders.S2STransformerBeamSearch modules: [!ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>] bos_index: !ref <bos_index> eos_index: !ref <eos_index> blank_index: !ref <blank_index> min_decode_ratio: !ref <min_decode_ratio> max_decode_ratio: !ref <max_decode_ratio> beam_size: !ref <test_beam_size> ctc_weight: !ref <ctc_weight_decode> lm_weight: !ref <lm_weight> lm_modules: !ref <lm_model> temperature: 1.25 temperature_lm: 1.25 using_eos_threshold: False length_normalization: True Tencoder: !new:speechbrain.lobes.models.transformer.TransformerASR.EncoderWrapper transformer: !ref <Transformer> encoder: !new:speechbrain.nnet.containers.LengthsCapableSequential input_shape: [null, null, !ref <n_mels>] compute_features: !ref <compute_features> normalize: !ref <normalizer> cnn: !ref <CNN> transformer_encoder: !ref <Tencoder> asr_model: !new:torch.nn.ModuleList - [!ref <normalizer>, !ref <CNN>, !ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>] log_softmax: !new:torch.nn.LogSoftmax dim: -1 compute_features: !new:speechbrain.lobes.features.Fbank sample_rate: !ref <sample_rate> n_fft: !ref <n_fft> n_mels: !ref <n_mels> modules: compute_features: !ref <compute_features> normalizer: !ref <normalizer> pre_transformer: !ref <CNN> transformer: !ref <Transformer> asr_model: !ref <asr_model> lm_model: !ref <lm_model> encoder: !ref <encoder> decoder: !ref <decoder> # The pretrainer allows a mapping between pretrained files and instances that # are declared in the yaml. pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer loadables: normalizer: !ref <normalizer> asr: !ref <asr_model> lm: !ref <lm_model> tokenizer: !ref <tokenizer>