Commit
•
4df5beb
1
Parent(s):
74b346d
Upload tokenizer_config.json with huggingface_hub
Browse files- tokenizer_config.json +12 -0
tokenizer_config.json
CHANGED
@@ -263834,9 +263834,21 @@
|
|
263834 |
"special": true
|
263835 |
}
|
263836 |
},
|
|
|
263837 |
"bos_token": "<|extra_203|>",
|
263838 |
"clean_up_tokenization_spaces": false,
|
|
|
|
|
263839 |
"eos_token": "<|extra_204|>",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
263840 |
"model_max_length": 1000000000000000019884624838656,
|
263841 |
"pad_token": "<|endoftext|>",
|
263842 |
"processor_class": "Emu3Processor",
|
|
|
263834 |
"special": true
|
263835 |
}
|
263836 |
},
|
263837 |
+
"boi_token": "<|image start|>",
|
263838 |
"bos_token": "<|extra_203|>",
|
263839 |
"clean_up_tokenization_spaces": false,
|
263840 |
+
"eof_token": "<|extra_201|>",
|
263841 |
+
"eoi_token": "<|image end|>",
|
263842 |
"eos_token": "<|extra_204|>",
|
263843 |
+
"extra_special_tokens": {
|
263844 |
+
"boi_token": "<|image start|>",
|
263845 |
+
"eof_token": "<|extra_201|>",
|
263846 |
+
"eoi_token": "<|image end|>",
|
263847 |
+
"image_token": "<image>",
|
263848 |
+
"image_wrapper_token": "<|image token|>"
|
263849 |
+
},
|
263850 |
+
"image_token": "<image>",
|
263851 |
+
"image_wrapper_token": "<|image token|>",
|
263852 |
"model_max_length": 1000000000000000019884624838656,
|
263853 |
"pad_token": "<|endoftext|>",
|
263854 |
"processor_class": "Emu3Processor",
|