update

tokenizer.py

@@ -1,11 +1,11 @@
 from transformers import PreTrainedTokenizerFast
 
 class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+
+    def prepare_for_model(self, ids, *args, **kwargs):
         breakpoint()
-        if token_ids_1 is None:
-            return [id for id in token_ids_0 if id != self.eos_token_id]
-        return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
+        filtered_ids = [id for id in ids if id != self.eos_token_id] 
+        return super().prepare_for_model(filtered_ids, *args, **kwargs)
 
 # Register the class
 from transformers import AutoTokenizer