Nanobit commited on
Commit
be22551
1 Parent(s): b832a0a

Fix unsupported operand type(s) for |

Browse files
Files changed (1) hide show
  1. src/axolotl/prompt_tokenizers.py +2 -2
src/axolotl/prompt_tokenizers.py CHANGED
@@ -4,7 +4,7 @@ import abc
4
  import copy
5
  import functools
6
  import logging
7
- from typing import Dict, List, Tuple
8
 
9
  from transformers import PreTrainedTokenizer
10
 
@@ -433,7 +433,7 @@ def parse_tokenized_to_result(
433
  current_len: int,
434
  res: Dict[str, List[int]],
435
  labels: list[int],
436
- pad_token_id: int | None = None,
437
  ) -> Tuple[Dict[str, List[int]], int]:
438
  """
439
  Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result
 
4
  import copy
5
  import functools
6
  import logging
7
+ from typing import Dict, List, Tuple, Union
8
 
9
  from transformers import PreTrainedTokenizer
10
 
 
433
  current_len: int,
434
  res: Dict[str, List[int]],
435
  labels: list[int],
436
+ pad_token_id: Union[int, None] = None,
437
  ) -> Tuple[Dict[str, List[int]], int]:
438
  """
439
  Parses the tokenized prompt and append the tokenized input_ids, attention_mask and labels to the result