RaushanTurganbay HF staff commited on
Commit
443d34d
1 Parent(s): 11675ef

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +5 -0
special_tokens_map.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "bos_token": {
3
  "content": "<|extra_203|>",
4
  "lstrip": false,
@@ -6,6 +7,8 @@
6
  "rstrip": false,
7
  "single_word": false
8
  },
 
 
9
  "eos_token": {
10
  "content": "<|extra_204|>",
11
  "lstrip": false,
@@ -13,6 +16,8 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
 
16
  "pad_token": {
17
  "content": "<|endoftext|>",
18
  "lstrip": false,
 
1
  {
2
+ "boi_token": "<|image start|>",
3
  "bos_token": {
4
  "content": "<|extra_203|>",
5
  "lstrip": false,
 
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
+ "eof_token": "<|extra_201|>",
11
+ "eoi_token": "<|image end|>",
12
  "eos_token": {
13
  "content": "<|extra_204|>",
14
  "lstrip": false,
 
16
  "rstrip": false,
17
  "single_word": false
18
  },
19
+ "image_token": "<image>",
20
+ "image_wrapper_token": "<|image token|>",
21
  "pad_token": {
22
  "content": "<|endoftext|>",
23
  "lstrip": false,