Update src/axolotl/utils/models.py
Browse filesCo-authored-by: NanoCode012 <kevinvong@rocketmail.com>
src/axolotl/utils/models.py
CHANGED
@@ -46,7 +46,7 @@ def load_tokenizer(
|
|
46 |
else:
|
47 |
tokenizer = AutoTokenizer.from_pretrained(
|
48 |
base_model_config,
|
49 |
-
trust_remote_code=
|
50 |
)
|
51 |
|
52 |
logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
|
|
|
46 |
else:
|
47 |
tokenizer = AutoTokenizer.from_pretrained(
|
48 |
base_model_config,
|
49 |
+
trust_remote_code=cfg.trust_remote_code or False,
|
50 |
)
|
51 |
|
52 |
logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
|