alfredplpl commited on
Commit
71a334b
1 Parent(s): dea81e3

Update tokenizer_config.json

Browse files

for https://github.com/oobabooga/text-generation-webui

Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -23,6 +23,6 @@
23
  "eos_token": "<|endoftext|>",
24
  "model_max_length": 1000000000000000019884624838656,
25
  "pad_token": "<|padding|>",
26
- "tokenizer_class": "GPTNeoXTokenizer",
27
  "unk_token": "<|endoftext|>"
28
  }
 
23
  "eos_token": "<|endoftext|>",
24
  "model_max_length": 1000000000000000019884624838656,
25
  "pad_token": "<|padding|>",
26
+ "tokenizer_class": "GPTNeoXTokenizerFast",
27
  "unk_token": "<|endoftext|>"
28
  }