Hugging Face's logo

dortucx
/
unkindtokenizer

distilbert
custom_code
Model card Files
xet
Community
dortucx commited on
Commit
72dfc2d
·
verified ·
1 Parent(s): 7b5a800

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "auto_map": {
3
- "AutoTokenizer": ["tokenization_yi.this_tokenizer_is_evil_actually", null]
4
  },
5
  "add_bos_token": false,
6
  "add_eos_token": false,
7
  "model_max_length": 4096,
8
- "tokenizer_class": "this_tokenizer_is_evil_actually"
9
  }
 
1
  {
2
  "auto_map": {
3
+ "AutoTokenizer": ["this_tokenizer_is_evil_actually.this_class_doesnt_exist_lol", null]
4
  },
5
  "add_bos_token": false,
6
  "add_eos_token": false,
7
  "model_max_length": 4096,
8
+ "tokenizer_class": "this_class_doesnt_exist_lol"
9
  }