oweller2
commited on
Commit
•
3608e05
1
Parent(s):
5a1ce8b
done
Browse files- tokenizer.py +3 -0
tokenizer.py
CHANGED
|
@@ -2,6 +2,9 @@ from transformers import PreTrainedTokenizer, AutoTokenizer
|
|
| 2 |
|
| 3 |
class ModernDecoderBERTTokenizer(PreTrainedTokenizer):
|
| 4 |
|
|
|
|
|
|
|
|
|
|
| 5 |
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
| 6 |
breakpoint()
|
| 7 |
if token_ids_1 is None:
|
|
|
|
| 2 |
|
| 3 |
class ModernDecoderBERTTokenizer(PreTrainedTokenizer):
|
| 4 |
|
| 5 |
+
def __init__(self, *args, **kwargs):
|
| 6 |
+
super().__init__(*args, **kwargs)
|
| 7 |
+
|
| 8 |
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
| 9 |
breakpoint()
|
| 10 |
if token_ids_1 is None:
|