hyperparams.yaml

# ############################################################################
# Model: Streaming E2E Conformer-Transducer ASR
# Encoder: Conformer
# Decoder: LSTM + greedy search
# Tokens: BPE with unigram
# losses: Transducer + CTC (optional) + CE (optional)
# Training: Librispeech 960h
# Authors:  Sylvain de Langen 2023, Titouan Parcollet 2023, Abdel HEBA, Mirco Ravanelli, Sung-Lin Yeh 2020
# ############################################################################

save_folder: !ref librispeech-streaming-conformer-transducer

# Training parameters
# To make Transformers converge, the global bath size should be large enough.
# The global batch size is computed as batch_size * n_gpus * grad_accumulation_factor.
# Empirically, we found that this value should be >= 128.
# Please, set your parameters accordingly.
number_of_epochs: 50
warmup_steps: 25000
num_workers: 4
batch_size_valid: 4
lr: 0.0008
weight_decay: 0.01
number_of_ctc_epochs: 40
ctc_weight: 0.3 # Multitask with CTC for the encoder (0.0 = disabled)
ce_weight: 0.0 # Multitask with CE for the decoder (0.0 = disabled)
max_grad_norm: 5.0
loss_reduction: 'batchmean'
precision: fp32 # bf16, fp16 or fp32

# The batch size is used if and only if dynamic batching is set to False
# Validation and testing are done with fixed batches and not dynamic batching.
batch_size: 8
grad_accumulation_factor: 4
sorting: ascending
avg_checkpoints: 10 # Number of checkpoints to average for evaluation

# Feature parameters
sample_rate: 16000
n_fft: 512
n_mels: 80
win_length: 32

# Streaming
streaming: True  # controls all Dynamic Chunk Training & chunk size & left context mechanisms

# This setup works well for 3090 24GB GPU, adapt it to your needs.
# Adjust grad_accumulation_factor depending on the DDP node count (here 3)
# Or turn it off (but training speed will decrease)
dynamic_batching: True
max_batch_len: 250
max_batch_len_val: 50 # we reduce it as the beam is much wider (VRAM)
num_bucket: 200

dynamic_batch_sampler:
   max_batch_len: !ref <max_batch_len>
   max_batch_len_val: !ref <max_batch_len_val>
   num_buckets: !ref <num_bucket>
   shuffle_ex: True # if true re-creates batches at each epoch shuffling examples.
   batch_ordering: random
   max_batch_ex: 256

# Model parameters
# Transformer
d_model: 512
joint_dim: 640
nhead: 8
num_encoder_layers: 12
num_decoder_layers: 0
d_ffn: 2048
transformer_dropout: 0.1
activation: !name:torch.nn.GELU
output_neurons: 1000
dec_dim: 512
dec_emb_dropout: 0.2
dec_dropout: 0.1

# Decoding parameters
blank_index: 0
bos_index: 0
eos_index: 0
pad_index: 0
beam_size: 10
nbest: 1
# by default {state,expand}_beam = 2.3 as mention in paper
# https://arxiv.org/abs/1904.02619
state_beam: 2.3
expand_beam: 2.3
lm_weight: 0.50

epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
   limit: !ref <number_of_epochs>

normalize: !new:speechbrain.processing.features.InputNormalization
   norm_type: global
   update_until_epoch: 4

compute_features: !new:speechbrain.lobes.features.Fbank
   sample_rate: !ref <sample_rate>
   n_fft: !ref <n_fft>
   n_mels: !ref <n_mels>
   win_length: !ref <win_length>

CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
   input_shape: (8, 10, 80)
   num_blocks: 2
   num_layers_per_block: 1
   out_channels: (64, 32)
   kernel_sizes: (3, 3)
   strides: (2, 2)
   residuals: (False, False)

Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length
   input_size: 640
   tgt_vocab: !ref <output_neurons>
   d_model: !ref <d_model>
   nhead: !ref <nhead>
   num_encoder_layers: !ref <num_encoder_layers>
   num_decoder_layers: !ref <num_decoder_layers>
   d_ffn: !ref <d_ffn>
   dropout: !ref <transformer_dropout>
   activation: !ref <activation>
   encoder_module: conformer
   attention_type: RelPosMHAXL
   normalize_before: True
   causal: False

# We must call an encoder wrapper so the decoder isn't run (we don't have any)
enc: !new:speechbrain.lobes.models.transformer.TransformerASR.EncoderWrapper
   transformer: !ref <Transformer>

# For MTL CTC over the encoder
proj_ctc: !new:speechbrain.nnet.linear.Linear
   input_size: !ref <joint_dim>
   n_neurons: !ref <output_neurons>

# Define some projection layers to make sure that enc and dec
# output dim are the same before joining
proj_enc: !new:speechbrain.nnet.linear.Linear
   input_size: !ref <d_model>
   n_neurons: !ref <joint_dim>
   bias: False

proj_dec: !new:speechbrain.nnet.linear.Linear
   input_size: !ref <dec_dim>
   n_neurons: !ref <joint_dim>
   bias: False

# Uncomment for MTL with CTC
ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
   blank_index: !ref <blank_index>
   reduction: !ref <loss_reduction>

emb: !new:speechbrain.nnet.embedding.Embedding
   num_embeddings: !ref <output_neurons>
   consider_as_one_hot: True
   blank_id: !ref <blank_index>

dec: !new:speechbrain.nnet.RNN.LSTM
   input_shape: [null, null, !ref <output_neurons> - 1]
   hidden_size: !ref <dec_dim>
   num_layers: 1
   re_init: True

Tjoint: !new:speechbrain.nnet.transducer.transducer_joint.Transducer_joint
   joint: sum # joint [sum | concat]
   nonlinearity: !ref <activation>

transducer_lin: !new:speechbrain.nnet.linear.Linear
   input_size: !ref <joint_dim>
   n_neurons: !ref <output_neurons>
   bias: False

log_softmax: !new:speechbrain.nnet.activations.Softmax
   apply_log: True

# for MTL
# update model if any HEAD module is added
modules:
   CNN: !ref <CNN>
   enc: !ref <enc>
   emb: !ref <emb>
   dec: !ref <dec>
   Tjoint: !ref <Tjoint>
   transducer_lin: !ref <transducer_lin>
   normalize: !ref <normalize>
   proj_ctc: !ref <proj_ctc>
   proj_dec: !ref <proj_dec>
   proj_enc: !ref <proj_enc>
#   dec_lin: !ref <dec_lin>

# for MTL
# update model if any HEAD module is added
model: !new:torch.nn.ModuleList
   - [!ref <CNN>, !ref <enc>, !ref <emb>, !ref <dec>, !ref <proj_enc>, !ref <proj_dec>, !ref <proj_ctc>, !ref <transducer_lin>]

# Tokenizer initialization
tokenizer: !new:sentencepiece.SentencePieceProcessor

Greedysearcher: !new:speechbrain.decoders.transducer.TransducerBeamSearcher
   decode_network_lst: [!ref <emb>, !ref <dec>, !ref <proj_dec>]
   tjoint: !ref <Tjoint>
   classifier_network: [!ref <transducer_lin>]
   blank_id: !ref <blank_index>
   beam_size: 1
   nbest: 1

Beamsearcher: !new:speechbrain.decoders.transducer.TransducerBeamSearcher
   decode_network_lst: [!ref <emb>, !ref <dec>, !ref <proj_dec>]
   tjoint: !ref <Tjoint>
   classifier_network: [!ref <transducer_lin>]
   blank_id: !ref <blank_index>
   beam_size: !ref <beam_size>
   nbest: !ref <nbest>
   # FIXME: when lm pretrained, use this
   # lm_module: !ref <lm_model>
   # lm_weight: !ref <lm_weight>
   state_beam: !ref <state_beam>
   expand_beam: !ref <expand_beam>

pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
   collect_in: !ref <save_folder>
   loadables:
      model: !ref <model>
      normalizer: !ref <normalize>
      tokenizer: !ref <tokenizer>

# inference stuff

make_tokenizer_streaming_context: !name:speechbrain.tokenizers.SentencePiece.SentencePieceDecoderStreamingContext
tokenizer_decode_streaming: !name:speechbrain.tokenizers.SentencePiece.spm_decode_preserve_leading_space

fea_streaming_extractor: !new:speechbrain.lobes.features.StreamingFeatureWrapper
   module: !new:speechbrain.lobes.models.convolution.ConformerFeatureExtractorWrapper
      - !ref <compute_features>
      - !ref <normalize>
      - !ref <CNN>
   # don't consider normalization as part of the input filter chain
   properties: !!python/object/apply:speechbrain.utils.filter_analysis.stack_filter_properties
      - [!ref <compute_features>, !ref <CNN>]