done

tokenizer.py

@@ -3,9 +3,10 @@ from transformers import PreTrainedTokenizerFast
 class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
 
     def _batch_encode_plus(self, *args, **kwargs):
-        breakpoint()
         outputs = super()._batch_encode_plus(*args, **kwargs)
-        outputs['input_ids'] = [[id for id in ids if id != self.eos_token_id] for ids in outputs['input_ids']]
+        del outputs["token_type_ids"]
+        for key in ['input_ids', 'attention_mask']:
+            outputs[key] = [sequence[:-1] for sequence in outputs[key]]
         return outputs
 
 # Register the class