RaushanTurganbay HF staff commited on
Commit
4df5beb
1 Parent(s): 74b346d

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +12 -0
tokenizer_config.json CHANGED
@@ -263834,9 +263834,21 @@
263834
  "special": true
263835
  }
263836
  },
 
263837
  "bos_token": "<|extra_203|>",
263838
  "clean_up_tokenization_spaces": false,
 
 
263839
  "eos_token": "<|extra_204|>",
 
 
 
 
 
 
 
 
 
263840
  "model_max_length": 1000000000000000019884624838656,
263841
  "pad_token": "<|endoftext|>",
263842
  "processor_class": "Emu3Processor",
 
263834
  "special": true
263835
  }
263836
  },
263837
+ "boi_token": "<|image start|>",
263838
  "bos_token": "<|extra_203|>",
263839
  "clean_up_tokenization_spaces": false,
263840
+ "eof_token": "<|extra_201|>",
263841
+ "eoi_token": "<|image end|>",
263842
  "eos_token": "<|extra_204|>",
263843
+ "extra_special_tokens": {
263844
+ "boi_token": "<|image start|>",
263845
+ "eof_token": "<|extra_201|>",
263846
+ "eoi_token": "<|image end|>",
263847
+ "image_token": "<image>",
263848
+ "image_wrapper_token": "<|image token|>"
263849
+ },
263850
+ "image_token": "<image>",
263851
+ "image_wrapper_token": "<|image token|>",
263852
  "model_max_length": 1000000000000000019884624838656,
263853
  "pad_token": "<|endoftext|>",
263854
  "processor_class": "Emu3Processor",