update

tokenizer.py

@@ -1,18 +1,7 @@
-from transformers import PreTrainedTokenizer, AutoTokenizer
-
-class ModernDecoderBERTTokenizer(PreTrainedTokenizer):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+from transformers import PreTrainedTokenizerFast
 
+class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        breakpoint()
         if token_ids_1 is None:
             return [id for id in token_ids_0 if id != self.eos_token_id]
         return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
-
-    def get_vocab(self):
-        breakpoint()
-        return dict(self.vocab.items())
-        
-AutoTokenizer.register("ModernDecoderBERTTokenizer", ModernDecoderBERTTokenizer)