Update tokenizer_config.json
Browse files- tokenizer_config.json +7 -1
tokenizer_config.json
CHANGED
@@ -17,5 +17,11 @@
|
|
17 |
"sep_token": "</s>",
|
18 |
"sp_model_kwargs": {},
|
19 |
"tokenizer_class": "Kosmos2Tokenizer",
|
20 |
-
"unk_token": "<unk>"
|
|
|
|
|
|
|
|
|
|
|
|
|
21 |
}
|
|
|
17 |
"sep_token": "</s>",
|
18 |
"sp_model_kwargs": {},
|
19 |
"tokenizer_class": "Kosmos2Tokenizer",
|
20 |
+
"unk_token": "<unk>",
|
21 |
+
"auto_map": {
|
22 |
+
"AutoTokenizer": [
|
23 |
+
"tokenization_kosmos2.Kosmos2Tokenizer",
|
24 |
+
"tokenization_kosmos2_fast.Kosmos2TokenizerFast"
|
25 |
+
]
|
26 |
+
},
|
27 |
}
|