Update tokenizer

Browse files

Files changed (2) hide show

tokenizer.json +0 -0
tokenizer_config.json +1 -1

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"bos_token": "<s>", "eos_token": "</s>", "~~unk_token~~": "~~<unk~~>", "~~sep_token~~": "</s>", "~~cls_token~~": "<s>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sp_model_kwargs": {}, "tokenizer_class": "XLMRobertaTokenizer"}

+ {"bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "cls_token": "<s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sp_model_kwargs": {}, "special_tokens_map_file": "/root/.cache/huggingface/transformers/6ea33bfad55f516eeb1e52bb6e3f5c198c976c6ebfaa7e1fb9f56b33b2ddc8a9.f6ac38e94db1fb749ad0f52af2f7cd5b7863b9b30f1aa680a1b0d22ec09b717d", "name_or_path": "AlekseyDorkin/xlm-roberta-large-en-ru", "tokenizer_class": "XLMRobertaTokenizer"}