Nanobit commited on
Commit
71bd062
1 Parent(s): cb9797e

Fix(tokenizer): Fix condition to add pad token (#477)

Browse files

* Fix(tokenizer): Fix condition to add pad token

* chore: fix lint

Files changed (1) hide show
  1. src/axolotl/utils/models.py +9 -4
src/axolotl/utils/models.py CHANGED
@@ -55,10 +55,15 @@ def load_tokenizer(cfg):
55
  **tokenizer_kwargs,
56
  )
57
 
58
- if tokenizer.__class__.__name__ in [
59
- "LlamaTokenizer",
60
- "LlamaTokenizerFast",
61
- ] and not hasattr(tokenizer, "pad_token"):
 
 
 
 
 
62
  # set a pad_token, but use eos_token so we don't add a new token
63
  tokenizer.pad_token = LLAMA_DEFAULT_EOS_TOKEN
64
 
 
55
  **tokenizer_kwargs,
56
  )
57
 
58
+ if (
59
+ tokenizer.__class__.__name__
60
+ in [
61
+ "LlamaTokenizer",
62
+ "LlamaTokenizerFast",
63
+ ]
64
+ and hasattr(tokenizer, "pad_token")
65
+ and not tokenizer.pad_token
66
+ ):
67
  # set a pad_token, but use eos_token so we don't add a new token
68
  tokenizer.pad_token = LLAMA_DEFAULT_EOS_TOKEN
69