from transformers import PreTrainedTokenizer, AutoTokenizer

class ModernDecoderBERTTokenizer(PreTrainedTokenizer):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        breakpoint()
        if token_ids_1 is None:
            return [id for id in token_ids_0 if id != self.eos_token_id]
        return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]

    def get_vocab(self):
        breakpoint()
        return dict(self.vocab.items())
        
AutoTokenizer.register("ModernDecoderBERTTokenizer", ModernDecoderBERTTokenizer)