oweller2
commited on
Commit
•
6a605a0
1
Parent(s):
970954b
update
Browse files- tokenizer.py +1 -1
tokenizer.py
CHANGED
|
@@ -8,4 +8,4 @@ class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
|
|
| 8 |
|
| 9 |
# Register the class
|
| 10 |
from transformers import AutoTokenizer
|
| 11 |
-
AutoTokenizer.register(ModernDecoderBERTTokenizer)
|
|
|
|
| 8 |
|
| 9 |
# Register the class
|
| 10 |
from transformers import AutoTokenizer
|
| 11 |
+
AutoTokenizer.register(ModernDecoderBERTTokenizer, fast_tokenizer_class=ModernDecoderBERTTokenizer)
|