This is a wrong solution, a new PR has been submitted, sorry...

#9
Files changed (1) hide show
  1. tokenizer.json +9 -0
tokenizer.json CHANGED
@@ -2306,6 +2306,15 @@
2306
  "rstrip": false,
2307
  "normalized": false,
2308
  "special": true
 
 
 
 
 
 
 
 
 
2309
  }
2310
  ],
2311
  "normalizer": null,
 
2306
  "rstrip": false,
2307
  "normalized": false,
2308
  "special": true
2309
+ },
2310
+ {
2311
+ "id": 128256,
2312
+ "content": "<|im_end|>",
2313
+ "single_word": false,
2314
+ "lstrip": false,
2315
+ "rstrip": false,
2316
+ "normalized": false,
2317
+ "special": true
2318
  }
2319
  ],
2320
  "normalizer": null,