intervitens commited on
Commit
774fb58
1 Parent(s): 0e7e002

Upload tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +4 -4
tokenizer.json CHANGED
@@ -59,7 +59,7 @@
59
  },
60
  {
61
  "id": 128006,
62
- "content": "<|im_start|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
@@ -86,7 +86,7 @@
86
  },
87
  {
88
  "id": 128009,
89
- "content": "<|im_end|>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
@@ -167,7 +167,7 @@
167
  },
168
  {
169
  "id": 128018,
170
- "content": "<|reserved_special_token_10|>",
171
  "single_word": false,
172
  "lstrip": false,
173
  "rstrip": false,
@@ -176,7 +176,7 @@
176
  },
177
  {
178
  "id": 128019,
179
- "content": "<|reserved_special_token_11|>",
180
  "single_word": false,
181
  "lstrip": false,
182
  "rstrip": false,
 
59
  },
60
  {
61
  "id": 128006,
62
+ "content": "<|start_header_id|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
 
86
  },
87
  {
88
  "id": 128009,
89
+ "content": "<|eot_id|>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
 
167
  },
168
  {
169
  "id": 128018,
170
+ "content": "<|im_start|>",
171
  "single_word": false,
172
  "lstrip": false,
173
  "rstrip": false,
 
176
  },
177
  {
178
  "id": 128019,
179
+ "content": "<|im_end|>",
180
  "single_word": false,
181
  "lstrip": false,
182
  "rstrip": false,