orionweller
/

test-flex-gpt

Model card Files Files and versions

orionweller commited on Jan 27, 2025

Commit

edca2d8

·

verified ·

1 Parent(s): 3901bef

Update tokenizer.py

Files changed (1) hide show

tokenizer.py +1 -1

tokenizer.py CHANGED Viewed

@@ -6,7 +6,7 @@ class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
     def _batch_encode_plus(self, *args, **kwargs):
         outputs = super()._batch_encode_plus(*args, **kwargs)
-        if "token_type_ids" in output:
             del outputs["token_type_ids"]
         # Get the input_ids to check for EOS tokens

     def _batch_encode_plus(self, *args, **kwargs):
         outputs = super()._batch_encode_plus(*args, **kwargs)
+        if "token_type_ids" in outputs:
             del outputs["token_type_ids"]
         # Get the input_ids to check for EOS tokens