oweller2
commited on
Commit
·
d77b85b
1
Parent(s):
6a605a0
done
Browse files- tokenizer.py +1 -0
tokenizer.py
CHANGED
|
@@ -2,6 +2,7 @@ from transformers import PreTrainedTokenizerFast
|
|
| 2 |
|
| 3 |
class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
|
| 4 |
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
|
|
|
| 5 |
if token_ids_1 is None:
|
| 6 |
return [id for id in token_ids_0 if id != self.eos_token_id]
|
| 7 |
return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
|
|
|
|
| 2 |
|
| 3 |
class ModernDecoderBERTTokenizer(PreTrainedTokenizerFast):
|
| 4 |
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
| 5 |
+
breakpoint()
|
| 6 |
if token_ids_1 is None:
|
| 7 |
return [id for id in token_ids_0 if id != self.eos_token_id]
|
| 8 |
return [id for id in token_ids_0 if id != self.eos_token_id] + [id for id in token_ids_1 if id != self.eos_token_id]
|