transfo-xl

/

transfo-xl-wt103

like 14

Follow

Transformer-XL community 4

kindly see the code snippet below. could see tokenizer.sym2idx defined, but tokenizer.idx2sym is an empty list.

>>> from transformers import TransfoXLTokenizer
>>> tokenizer = TransfoXLTokenizer.from_pretrained("transfo-xl-wt103")

>>> enc = tokenizer.encode("Hello, my dog is cute")
>>> enc
[14049, 2, 617, 3225, 23, 16072]

>>> tokenizer.decode
<bound method PreTrainedTokenizerBase.decode of TransfoXLTokenizer(name_or_path='transfo-xl-wt103', vocab_size=0, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'eos_token': '<eos>', 'unk_token': '<unk>', 'additional_special_tokens': ['<formula>']}, clean_up_tokenization_spaces=True),  added_tokens_decoder={
    0: AddedToken("<eos>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
    24: AddedToken("<unk>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
    3039: AddedToken("<formula>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
}>
>>> tokenizer.decode(enc)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/home/.local/lib/python3.8/site-packages/transformers/tokenization_utils_base.py", line 3738, in decode
    return self._decode(
  File "/home/home/.local/lib/python3.8/site-packages/transformers/tokenization_utils.py", line 1001, in _decode
    filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
  File "/home/home/.local/lib/python3.8/site-packages/transformers/tokenization_utils.py", line 982, in convert_ids_to_tokens
    tokens.append(self._convert_id_to_token(index))
  File "/home/home/.local/lib/python3.8/site-packages/transformers/models/transfo_xl/tokenization_transfo_xl.py", line 451, in _convert_id_to_token
    return self.idx2sym[idx]
IndexError: list index out of range