feat: add fast tokenizer

* also fix tokenizer_config

Files changed (2) hide show

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -11,5 +11,6 @@
   "eos_token": "[SEP]",
   "tokenize_chinese_chars": true,
   "strip_accents": null,
-  "model_max_length": 512
 }

   "eos_token": "[SEP]",
   "tokenize_chinese_chars": true,
   "strip_accents": null,
+  "model_max_length": 512,
+  "tokenizer_class": "BertTokenizer"
 }