don't resize embeddings if it's already large enough (#577)
Browse files* don't resize embeddings if it's already large enough
* make sure to tie weights, even if we aren't resizing
src/axolotl/utils/models.py
CHANGED
@@ -302,7 +302,10 @@ def load_model(
|
|
302 |
if cfg.resize_token_embeddings_to_32x
|
303 |
else len(tokenizer)
|
304 |
)
|
305 |
-
model.
|
|
|
|
|
|
|
306 |
|
307 |
if (
|
308 |
hasattr(model.config, "max_position_embeddings")
|
|
|
302 |
if cfg.resize_token_embeddings_to_32x
|
303 |
else len(tokenizer)
|
304 |
)
|
305 |
+
if model.get_input_embeddings().num_embeddings < embeddings_len:
|
306 |
+
model.resize_token_embeddings(embeddings_len)
|
307 |
+
else:
|
308 |
+
model.tie_weights()
|
309 |
|
310 |
if (
|
311 |
hasattr(model.config, "max_position_embeddings")
|