Hezar: Upload tokenizer_config.yaml

preprocessor/tokenizer_config.yaml

@@ -1,19 +1,25 @@
 name: bpe_tokenizer
 config_type: preprocessor
+max_length: 512
 truncation_strategy: longest_first
 truncation_direction: right
+stride: 0
 padding_strategy: longest
 padding_direction: right
-pad_token_id: 0
-pad_token: <pad>
+pad_to_multiple_of: 0
 pad_token_type_id: 0
+bos_token: <s>
+eos_token: </s>
+unk_token: <unk>
+sep_token: <sep>
+pad_token: <pad>
+cls_token: <cls>
+mask_token: <mask>
 continuing_subword_prefix: ''
 end_of_word_suffix: ''
 fuse_unk: false
-train_config:
-  name: bpe_tokenizer
-  config_type: preprocessor
-  vocab_size: 30000
-  min_frequency: 2
-  limit_alphabet: 1000
-  show_progress: true
+vocab_size: 42000
+min_frequency: 2
+limit_alphabet: 1000
+initial_alphabet: []
+show_progress: true