arxyzan commited on
Commit
c32df6e
1 Parent(s): 54b8211

Hezar: Upload tokenizer and config

Browse files
Files changed (1) hide show
  1. preprocessor/tokenizer.json +1 -0
preprocessor/tokenizer.json CHANGED
@@ -1119,6 +1119,7 @@
1119
  "continuing_subword_prefix": "",
1120
  "end_of_word_suffix": "",
1121
  "fuse_unk": false,
 
1122
  "vocab": {
1123
  "!": 0,
1124
  "\"": 1,
 
1119
  "continuing_subword_prefix": "",
1120
  "end_of_word_suffix": "",
1121
  "fuse_unk": false,
1122
+ "byte_fallback": false,
1123
  "vocab": {
1124
  "!": 0,
1125
  "\"": 1,