update

tokenizer.py

@@ -2,10 +2,11 @@ from transformers import PreTrainedTokenizerFast
 
 class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
 
-    def prepare_for_model(self, ids, *args, **kwargs):
+    def _batch_encode_plus(self, *args, **kwargs):
         breakpoint()
-        filtered_ids = [id for id in ids if id != self.eos_token_id] 
-        return super().prepare_for_model(filtered_ids, *args, **kwargs)
+        outputs = super()._batch_encode_plus(*args, **kwargs)
+        outputs['input_ids'] = [[id for id in ids if id != self.eos_token_id] for ids in outputs['input_ids']]
+        return outputs
 
 # Register the class
 from transformers import AutoTokenizer