Valentin De Matos
commited on
fix: handles AutoTokenizer from untrusted source
Browse filesSet trust_remote_code param depending of cfg.trust_remote_code when calling AutoTokenizer.from_pretrained
src/axolotl/utils/models.py
CHANGED
@@ -167,7 +167,7 @@ def load_model(
|
|
167 |
else:
|
168 |
tokenizer = getattr(transformers, tokenizer_type).from_pretrained(model)
|
169 |
except:
|
170 |
-
tokenizer = AutoTokenizer.from_pretrained(base_model_config)
|
171 |
|
172 |
logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
|
173 |
logging.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")
|
|
|
167 |
else:
|
168 |
tokenizer = getattr(transformers, tokenizer_type).from_pretrained(model)
|
169 |
except:
|
170 |
+
tokenizer = AutoTokenizer.from_pretrained(base_model_config, trust_remote_code=True if cfg.trust_remote_code is True else False)
|
171 |
|
172 |
logging.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
|
173 |
logging.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")
|