Hezar: Upload tokenizer_config.yaml

preprocessor/tokenizer_config.yaml

@@ -1,4 +1,4 @@
-name: wordpiece_tokenizer
+name: bpe_tokenizer
 config_type: preprocessor
 max_length: 512
 truncation_strategy: longest_first
@@ -8,12 +8,16 @@ padding_strategy: longest
 padding_direction: right
 pad_to_multiple_of: 0
 pad_token_type_id: 0
-unk_token: '[UNK]'
-sep_token: '[SEP]'
-pad_token: '[PAD]'
-cls_token: '[CLS]'
-mask_token: '[MASK]'
-wordpieces_prefix: '##'
+bos_token: <s>
+eos_token: </s>
+unk_token: <unk>
+sep_token: <sep>
+pad_token: <pad>
+cls_token: <cls>
+mask_token: <mask>
+continuing_subword_prefix: ''
+end_of_word_suffix: ''
+fuse_unk: false
 vocab_size: 42000
 min_frequency: 2
 limit_alphabet: 1000