winglian commited on
Commit
3607882
1 Parent(s): 12a2dbb

don't resize embeddings if it's already large enough (#577)

Browse files

* don't resize embeddings if it's already large enough

* make sure to tie weights, even if we aren't resizing

Files changed (1) hide show
  1. src/axolotl/utils/models.py +4 -1
src/axolotl/utils/models.py CHANGED
@@ -302,7 +302,10 @@ def load_model(
302
  if cfg.resize_token_embeddings_to_32x
303
  else len(tokenizer)
304
  )
305
- model.resize_token_embeddings(embeddings_len)
 
 
 
306
 
307
  if (
308
  hasattr(model.config, "max_position_embeddings")
 
302
  if cfg.resize_token_embeddings_to_32x
303
  else len(tokenizer)
304
  )
305
+ if model.get_input_embeddings().num_embeddings < embeddings_len:
306
+ model.resize_token_embeddings(embeddings_len)
307
+ else:
308
+ model.tie_weights()
309
 
310
  if (
311
  hasattr(model.config, "max_position_embeddings")