AnonY0324 commited on
Commit
72c1f63
·
verified ·
1 Parent(s): b344d37

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +1 -0
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}, "additional_special_tokens": ["<s>", "</s>", "<2as>", "<2bn>", "<2en>", "<2gu>", "<2hi>", "<2kn>", "<2ml>", "<2mr>", "<2or>", "<2pa>", "<2ta>", "<2te>"]}