|
[WARNING|parser.py:279] 2024-08-31 19:14:40,267 >> We recommend enable `upcast_layernorm` in quantized training.
|
|
|
|
[INFO|parser.py:351] 2024-08-31 19:14:40,267 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16
|
|
|
|
[INFO|tokenization_utils_base.py:2289] 2024-08-31 19:14:40,450 >> loading file tokenizer.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\tokenizer.json
|
|
|
|
[INFO|tokenization_utils_base.py:2289] 2024-08-31 19:14:40,450 >> loading file added_tokens.json from cache at None
|
|
|
|
[INFO|tokenization_utils_base.py:2289] 2024-08-31 19:14:40,450 >> loading file special_tokens_map.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\special_tokens_map.json
|
|
|
|
[INFO|tokenization_utils_base.py:2289] 2024-08-31 19:14:40,451 >> loading file tokenizer_config.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\tokenizer_config.json
|
|
|
|
[INFO|tokenization_utils_base.py:2533] 2024-08-31 19:14:40,616 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
|
|
|
|
[INFO|template.py:373] 2024-08-31 19:14:40,616 >> Add pad token: <|eot_id|>
|
|
|
|
[INFO|loader.py:52] 2024-08-31 19:14:40,617 >> Loading dataset qa-unc-dpo.json...
|
|
|
|
[INFO|configuration_utils.py:733] 2024-08-31 19:14:59,967 >> loading configuration file config.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\config.json
|
|
|
|
[INFO|configuration_utils.py:800] 2024-08-31 19:14:59,968 >> Model config LlamaConfig {
|
|
"_name_or_path": "NousResearch/Meta-Llama-3.1-8B-Instruct",
|
|
"architectures": [
|
|
"LlamaForCausalLM"
|
|
],
|
|
"attention_bias": false,
|
|
"attention_dropout": 0.0,
|
|
"bos_token_id": 128000,
|
|
"eos_token_id": [
|
|
128001,
|
|
128008,
|
|
128009
|
|
],
|
|
"hidden_act": "silu",
|
|
"hidden_size": 4096,
|
|
"initializer_range": 0.02,
|
|
"intermediate_size": 14336,
|
|
"max_position_embeddings": 131072,
|
|
"mlp_bias": false,
|
|
"model_type": "llama",
|
|
"num_attention_heads": 32,
|
|
"num_hidden_layers": 32,
|
|
"num_key_value_heads": 8,
|
|
"pretraining_tp": 1,
|
|
"rms_norm_eps": 1e-05,
|
|
"rope_scaling": {
|
|
"factor": 8.0,
|
|
"high_freq_factor": 4.0,
|
|
"low_freq_factor": 1.0,
|
|
"original_max_position_embeddings": 8192,
|
|
"rope_type": "llama3"
|
|
},
|
|
"rope_theta": 500000.0,
|
|
"tie_word_embeddings": false,
|
|
"torch_dtype": "bfloat16",
|
|
"transformers_version": "4.43.3",
|
|
"use_cache": true,
|
|
"vocab_size": 128256
|
|
}
|
|
|
|
|
|
[WARNING|rope.py:57] 2024-08-31 19:14:59,970 >> Input length is smaller than max length. Consider increase input length.
|
|
|
|
[INFO|rope.py:63] 2024-08-31 19:14:59,970 >> Using linear scaling strategy and setting scaling factor to 1.0
|
|
|
|
[INFO|quantization.py:182] 2024-08-31 19:14:59,971 >> Quantizing model to 4 bit with bitsandbytes.
|
|
|
|
[INFO|modeling_utils.py:3634] 2024-08-31 19:15:00,414 >> loading weights file model.safetensors from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\model.safetensors.index.json
|
|
|
|
[INFO|modeling_utils.py:1572] 2024-08-31 19:15:00,426 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
|
|
|
|
[INFO|configuration_utils.py:1038] 2024-08-31 19:15:00,429 >> Generate config GenerationConfig {
|
|
"bos_token_id": 128000,
|
|
"eos_token_id": [
|
|
128001,
|
|
128008,
|
|
128009
|
|
]
|
|
}
|
|
|
|
|
|
[INFO|modeling_utils.py:4463] 2024-08-31 19:15:57,314 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
|
|
|
|
|
|
[INFO|modeling_utils.py:4471] 2024-08-31 19:15:57,314 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at NousResearch/Meta-Llama-3.1-8B-Instruct.
|
|
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
|
|
|
|
[INFO|configuration_utils.py:993] 2024-08-31 19:15:57,500 >> loading configuration file generation_config.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\generation_config.json
|
|
|
|
[INFO|configuration_utils.py:1038] 2024-08-31 19:15:57,502 >> Generate config GenerationConfig {
|
|
"bos_token_id": 128000,
|
|
"do_sample": true,
|
|
"eos_token_id": [
|
|
128001,
|
|
128008,
|
|
128009
|
|
],
|
|
"temperature": 0.6,
|
|
"top_p": 0.9
|
|
}
|
|
|
|
|
|
[WARNING|quantizer_bnb_4bit.py:305] 2024-08-31 19:15:57,597 >> You are calling `save_pretrained` to a 4-bit converted model, but your `bitsandbytes` version doesn't support it. If you want to save 4-bit models, make sure to have `bitsandbytes>=0.41.3` installed.
|
|
|
|
[INFO|checkpointing.py:103] 2024-08-31 19:15:57,598 >> Gradient checkpointing enabled.
|
|
|
|
[INFO|attention.py:82] 2024-08-31 19:15:57,598 >> Using FlashAttention-2 for faster training and inference.
|
|
|
|
[INFO|adapter.py:302] 2024-08-31 19:15:57,598 >> Upcasting trainable params to float32.
|
|
|
|
[INFO|adapter.py:158] 2024-08-31 19:15:57,598 >> Fine-tuning method: LoRA
|
|
|
|
[INFO|adapter.py:203] 2024-08-31 19:15:58,444 >> Loaded adapter(s): saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP
|
|
|
|
[INFO|loader.py:196] 2024-08-31 19:15:58,454 >> trainable params: 83,886,080 || all params: 8,114,147,328 || trainable%: 1.0338
|
|
|
|
[INFO|trainer.py:648] 2024-08-31 19:15:58,522 >> Using auto half precision backend
|
|
|
|
[INFO|trainer.py:2134] 2024-08-31 19:15:58,646 >> ***** Running training *****
|
|
|
|
[INFO|trainer.py:2135] 2024-08-31 19:15:58,646 >> Num examples = 83
|
|
|
|
[INFO|trainer.py:2136] 2024-08-31 19:15:58,646 >> Num Epochs = 1
|
|
|
|
[INFO|trainer.py:2137] 2024-08-31 19:15:58,647 >> Instantaneous batch size per device = 1
|
|
|
|
[INFO|trainer.py:2140] 2024-08-31 19:15:58,647 >> Total train batch size (w. parallel, distributed & accumulation) = 1
|
|
|
|
[INFO|trainer.py:2141] 2024-08-31 19:15:58,647 >> Gradient Accumulation steps = 1
|
|
|
|
[INFO|trainer.py:2142] 2024-08-31 19:15:58,647 >> Total optimization steps = 83
|
|
|
|
[INFO|trainer.py:2143] 2024-08-31 19:15:58,649 >> Number of trainable parameters = 83,886,080
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:06,578 >> {'loss': 1.4781, 'learning_rate': 4.8230e-05, 'epoch': 0.12, 'throughput': 1118.38}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:13,176 >> {'loss': 1.4732, 'learning_rate': 4.3172e-05, 'epoch': 0.24, 'throughput': 1219.49}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:19,792 >> {'loss': 1.3140, 'learning_rate': 3.5542e-05, 'epoch': 0.36, 'throughput': 1301.78}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:25,955 >> {'loss': 1.2266, 'learning_rate': 2.6419e-05, 'epoch': 0.48, 'throughput': 1322.02}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:31,576 >> {'loss': 1.0201, 'learning_rate': 1.7095e-05, 'epoch': 0.60, 'throughput': 1319.87}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:37,779 >> {'loss': 1.4448, 'learning_rate': 8.8901e-06, 'epoch': 0.72, 'throughput': 1318.76}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:45,117 >> {'loss': 1.1615, 'learning_rate': 2.9659e-06, 'epoch': 0.84, 'throughput': 1347.76}
|
|
|
|
[INFO|callbacks.py:320] 2024-08-31 19:16:52,319 >> {'loss': 1.2626, 'learning_rate': 1.6100e-07, 'epoch': 0.96, 'throughput': 1363.67}
|
|
|
|
[INFO|trainer.py:3503] 2024-08-31 19:16:54,139 >> Saving model checkpoint to saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP-DPO\checkpoint-83
|
|
|
|
[INFO|configuration_utils.py:733] 2024-08-31 19:16:54,398 >> loading configuration file config.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\config.json
|
|
|
|
[INFO|configuration_utils.py:800] 2024-08-31 19:16:54,401 >> Model config LlamaConfig {
|
|
"architectures": [
|
|
"LlamaForCausalLM"
|
|
],
|
|
"attention_bias": false,
|
|
"attention_dropout": 0.0,
|
|
"bos_token_id": 128000,
|
|
"eos_token_id": [
|
|
128001,
|
|
128008,
|
|
128009
|
|
],
|
|
"hidden_act": "silu",
|
|
"hidden_size": 4096,
|
|
"initializer_range": 0.02,
|
|
"intermediate_size": 14336,
|
|
"max_position_embeddings": 131072,
|
|
"mlp_bias": false,
|
|
"model_type": "llama",
|
|
"num_attention_heads": 32,
|
|
"num_hidden_layers": 32,
|
|
"num_key_value_heads": 8,
|
|
"pretraining_tp": 1,
|
|
"rms_norm_eps": 1e-05,
|
|
"rope_scaling": {
|
|
"factor": 8.0,
|
|
"high_freq_factor": 4.0,
|
|
"low_freq_factor": 1.0,
|
|
"original_max_position_embeddings": 8192,
|
|
"rope_type": "llama3"
|
|
},
|
|
"rope_theta": 500000.0,
|
|
"tie_word_embeddings": false,
|
|
"torch_dtype": "bfloat16",
|
|
"transformers_version": "4.43.3",
|
|
"use_cache": true,
|
|
"vocab_size": 128256
|
|
}
|
|
|
|
|
|
[INFO|tokenization_utils_base.py:2702] 2024-08-31 19:16:54,664 >> tokenizer config file saved in saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP-DPO\checkpoint-83\tokenizer_config.json
|
|
|
|
[INFO|tokenization_utils_base.py:2711] 2024-08-31 19:16:54,664 >> Special tokens file saved in saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP-DPO\checkpoint-83\special_tokens_map.json
|
|
|
|
[INFO|trainer.py:2394] 2024-08-31 19:16:54,983 >>
|
|
|
|
Training completed. Do not forget to share your model on huggingface.co/models =)
|
|
|
|
|
|
|
|
[INFO|trainer.py:3503] 2024-08-31 19:16:54,987 >> Saving model checkpoint to saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP-DPO
|
|
|
|
[INFO|configuration_utils.py:733] 2024-08-31 19:16:55,209 >> loading configuration file config.json from cache at G:\Dataset\cache\hub\models--NousResearch--Meta-Llama-3.1-8B-Instruct\snapshots\d10aef7999a2b5ba950ab3974312feeedbfe0b77\config.json
|
|
|
|
[INFO|configuration_utils.py:800] 2024-08-31 19:16:55,210 >> Model config LlamaConfig {
|
|
"architectures": [
|
|
"LlamaForCausalLM"
|
|
],
|
|
"attention_bias": false,
|
|
"attention_dropout": 0.0,
|
|
"bos_token_id": 128000,
|
|
"eos_token_id": [
|
|
128001,
|
|
128008,
|
|
128009
|
|
],
|
|
"hidden_act": "silu",
|
|
"hidden_size": 4096,
|
|
"initializer_range": 0.02,
|
|
"intermediate_size": 14336,
|
|
"max_position_embeddings": 131072,
|
|
"mlp_bias": false,
|
|
"model_type": "llama",
|
|
"num_attention_heads": 32,
|
|
"num_hidden_layers": 32,
|
|
"num_key_value_heads": 8,
|
|
"pretraining_tp": 1,
|
|
"rms_norm_eps": 1e-05,
|
|
"rope_scaling": {
|
|
"factor": 8.0,
|
|
"high_freq_factor": 4.0,
|
|
"low_freq_factor": 1.0,
|
|
"original_max_position_embeddings": 8192,
|
|
"rope_type": "llama3"
|
|
},
|
|
"rope_theta": 500000.0,
|
|
"tie_word_embeddings": false,
|
|
"torch_dtype": "bfloat16",
|
|
"transformers_version": "4.43.3",
|
|
"use_cache": true,
|
|
"vocab_size": 128256
|
|
}
|
|
|
|
|
|
[INFO|tokenization_utils_base.py:2702] 2024-08-31 19:17:02,183 >> tokenizer config file saved in saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP-DPO\tokenizer_config.json
|
|
|
|
[INFO|tokenization_utils_base.py:2711] 2024-08-31 19:17:02,184 >> Special tokens file saved in saves\LLaMA3.1-8B-Chat\lora\Luminia-8B-RP-DPO\special_tokens_map.json
|
|
|
|
[WARNING|ploting.py:89] 2024-08-31 19:17:02,350 >> No metric eval_loss to plot.
|
|
|
|
[INFO|modelcard.py:449] 2024-08-31 19:17:02,400 >> Dropping the following result as it does not have all the necessary fields:
|
|
{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
|
|
|
|
|