Upload BertForPrefixMarking.py
Browse files- BertForPrefixMarking.py +1 -0
BertForPrefixMarking.py
CHANGED
|
@@ -174,6 +174,7 @@ def encode_sentences_for_bert_for_prefix_marking(tokenizer: BertTokenizerFast, s
|
|
| 174 |
next_tok_idx = tok_idx + 1
|
| 175 |
while next_tok_idx < len(tokens) and tokens[next_tok_idx].startswith('##'):
|
| 176 |
token += tokens[next_tok_idx][2:]
|
|
|
|
| 177 |
|
| 178 |
# find all the possible prefixes - and mark them as 0 (and in the possible mark it as it's value for embed lookup)
|
| 179 |
for pre_class in get_prefix_classes_from_str(token):
|
|
|
|
| 174 |
next_tok_idx = tok_idx + 1
|
| 175 |
while next_tok_idx < len(tokens) and tokens[next_tok_idx].startswith('##'):
|
| 176 |
token += tokens[next_tok_idx][2:]
|
| 177 |
+
next_tok_idx += 1
|
| 178 |
|
| 179 |
# find all the possible prefixes - and mark them as 0 (and in the possible mark it as it's value for embed lookup)
|
| 180 |
for pre_class in get_prefix_classes_from_str(token):
|