oweller2
commited on
Commit
·
d776f8b
1
Parent(s):
8b0f7b6
add tokenizer
Browse files- tokenizer.py +3 -5
tokenizer.py
CHANGED
|
@@ -1,9 +1,7 @@
|
|
| 1 |
-
from transformers import PreTrainedTokenizer, AutoTokenizer
|
|
|
|
|
|
|
| 2 |
|
| 3 |
-
class ModernDecoderBERTTokenizer(GPT2Tokenizer):
|
| 4 |
-
def __init__(self, *args, **kwargs):
|
| 5 |
-
super().__init__(*args, **kwargs)
|
| 6 |
-
|
| 7 |
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
| 8 |
breakpoint()
|
| 9 |
if token_ids_1 is None:
|
|
|
|
| 1 |
+
from transformers import PreTrainedTokenizer, AutoTokenizer
|
| 2 |
+
|
| 3 |
+
class ModernDecoderBERTTokenizer(PreTrainedTokenizer):
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
| 6 |
breakpoint()
|
| 7 |
if token_ids_1 is None:
|