name: bpe_tokenizer
config_type: preprocessor
truncation_strategy: no_truncation
padding_strategy: no_padding
continuing_subword_prefix: ''
end_of_word_suffix: ''
fuse_unk: false
train_config:
  name: bpe_tokenizer
  config_type: preprocessor
  vocab_size: 30000
  min_frequency: 2
  limit_alphabet: 1000
  show_progress: true