mahnerak commited on
Commit
e911345
1 Parent(s): 55bfe5b

Fix the arguments

Browse files
Files changed (1) hide show
  1. tokenizer.json +41 -3
tokenizer.json CHANGED
@@ -216,11 +216,49 @@
216
  "post_processor": {
217
  "type": "TemplateProcessing",
218
  "single": [
219
- { "SpecialToken": { "id": "<|begin_of_text|>", "type_id": 0 } },
220
- { "Sequence": { "id": "A", "type_id": 0 } }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
  ],
222
  "special_tokens": {
223
- "<|begin_of_text|>": { "id": "<|begin_of_text|>", "ids": [100256], "tokens": ["<|begin_of_text|>"] }
 
 
 
 
 
 
 
 
224
  }
225
  },
226
  "decoder": {
 
216
  "post_processor": {
217
  "type": "TemplateProcessing",
218
  "single": [
219
+ {
220
+ "SpecialToken": {
221
+ "id": "<|begin_of_text|>",
222
+ "type_id": 0
223
+ }
224
+ },
225
+ {
226
+ "Sequence": {
227
+ "id": "A",
228
+ "type_id": 0
229
+ }
230
+ }
231
+ ],
232
+ "pair": [
233
+ {
234
+ "SpecialToken": {
235
+ "id": "<|begin_of_text|>",
236
+ "type_id": 0
237
+ }
238
+ },
239
+ {
240
+ "Sequence": {
241
+ "id": "A",
242
+ "type_id": 0
243
+ }
244
+ },
245
+ {
246
+ "Sequence": {
247
+ "id": "B",
248
+ "type_id": 1
249
+ }
250
+ }
251
  ],
252
  "special_tokens": {
253
+ "<|begin_of_text|>": {
254
+ "id": "<|begin_of_text|>",
255
+ "ids": [
256
+ 100256
257
+ ],
258
+ "tokens": [
259
+ "<|begin_of_text|>"
260
+ ]
261
+ }
262
  }
263
  },
264
  "decoder": {