intervitens
commited on
Commit
•
774fb58
1
Parent(s):
0e7e002
Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +4 -4
tokenizer.json
CHANGED
@@ -59,7 +59,7 @@
|
|
59 |
},
|
60 |
{
|
61 |
"id": 128006,
|
62 |
-
"content": "<|
|
63 |
"single_word": false,
|
64 |
"lstrip": false,
|
65 |
"rstrip": false,
|
@@ -86,7 +86,7 @@
|
|
86 |
},
|
87 |
{
|
88 |
"id": 128009,
|
89 |
-
"content": "<|
|
90 |
"single_word": false,
|
91 |
"lstrip": false,
|
92 |
"rstrip": false,
|
@@ -167,7 +167,7 @@
|
|
167 |
},
|
168 |
{
|
169 |
"id": 128018,
|
170 |
-
"content": "<|
|
171 |
"single_word": false,
|
172 |
"lstrip": false,
|
173 |
"rstrip": false,
|
@@ -176,7 +176,7 @@
|
|
176 |
},
|
177 |
{
|
178 |
"id": 128019,
|
179 |
-
"content": "<|
|
180 |
"single_word": false,
|
181 |
"lstrip": false,
|
182 |
"rstrip": false,
|
|
|
59 |
},
|
60 |
{
|
61 |
"id": 128006,
|
62 |
+
"content": "<|start_header_id|>",
|
63 |
"single_word": false,
|
64 |
"lstrip": false,
|
65 |
"rstrip": false,
|
|
|
86 |
},
|
87 |
{
|
88 |
"id": 128009,
|
89 |
+
"content": "<|eot_id|>",
|
90 |
"single_word": false,
|
91 |
"lstrip": false,
|
92 |
"rstrip": false,
|
|
|
167 |
},
|
168 |
{
|
169 |
"id": 128018,
|
170 |
+
"content": "<|im_start|>",
|
171 |
"single_word": false,
|
172 |
"lstrip": false,
|
173 |
"rstrip": false,
|
|
|
176 |
},
|
177 |
{
|
178 |
"id": 128019,
|
179 |
+
"content": "<|im_end|>",
|
180 |
"single_word": false,
|
181 |
"lstrip": false,
|
182 |
"rstrip": false,
|