Hezar: Upload tokenizer_config.yaml

preprocessor/tokenizer_config.yaml

@@ -1,4 +1,4 @@
-name: bpe_tokenizer
+name: wordpiece_tokenizer
 config_type: preprocessor
 max_length: 512
 truncation_strategy: longest_first
@@ -8,16 +8,12 @@ padding_strategy: longest
 padding_direction: right
 pad_to_multiple_of: 0
 pad_token_type_id: 0
-bos_token: <s>
-eos_token: </s>
-unk_token: <unk>
-sep_token: <sep>
-pad_token: <pad>
-cls_token: <cls>
-mask_token: <mask>
-continuing_subword_prefix: ''
-end_of_word_suffix: ''
-fuse_unk: false
+unk_token: '[UNK]'
+sep_token: '[SEP]'
+pad_token: '[PAD]'
+cls_token: '[CLS]'
+mask_token: '[MASK]'
+wordpieces_prefix: '##'
 vocab_size: 42000
 min_frequency: 2
 limit_alphabet: 1000