Model save

README.md

@@ -1,14 +1,12 @@
 ---
 license: apache-2.0
-library_name: peft
+base_model: mistralai/Mistral-7B-Instruct-v0.2
 tags:
-- alignment-handbook
 - trl
 - sft
 - generated_from_trainer
-base_model: mistralai/Mistral-7B-Instruct-v0.2
 datasets:
-- preference-data
+- generator
 model-index:
 - name: feedback_p0.1_seed42_level2_rare
   results: []
@@ -19,9 +17,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # feedback_p0.1_seed42_level2_rare
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the preference-data dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.8673
+- Loss: 0.2977
 
 ## Model description
 
@@ -41,13 +39,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 1e-05
-- train_batch_size: 8
+- train_batch_size: 2
 - eval_batch_size: 1
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
-- gradient_accumulation_steps: 2
-- total_train_batch_size: 64
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
 - total_eval_batch_size: 4
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -58,13 +56,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 0.853         | 0.9988 | 423  | 0.8673          |
+| 0.346         | 0.9997 | 931  | 0.2977          |
 
 
 ### Framework versions
 
-- PEFT 0.11.1
 - Transformers 4.43.4
 - Pytorch 2.3.1+cu121
 - Datasets 2.19.1
-- Tokenizers 0.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -1,14 +1,9 @@
 {
-    "epoch": 0.9988193624557261,
-    "eval_loss": 0.8673050403594971,
-    "eval_runtime": 2.0994,
-    "eval_samples": 10,
-    "eval_samples_per_second": 0.953,
-    "eval_steps_per_second": 0.476,
-    "total_flos": 1701364321222656.0,
+    "epoch": 0.9997315436241611,
+    "total_flos": 194880114524160.0,
     "train_loss": 0.0,
-    "train_runtime": 0.0113,
-    "train_samples": 90047,
-    "train_samples_per_second": 2398604.87,
-    "train_steps_per_second": 37460.213
+    "train_runtime": 0.0085,
+    "train_samples": 98952,
+    "train_samples_per_second": 3497894.361,
+    "train_steps_per_second": 109279.854
 }

config.json

@@ -22,6 +22,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.43.4",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.43.4"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23d4d60e4f957c3594368f739562fe754e8ea1256c13894f088683c487b6c7e0
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cce15fa203f380cff1d01841cb6e38cf8f8a7a7f36c064132891a4f399756a34
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6242f75d0617ab7a77e147861c4853cd980e50e6068b4dc5c75dec64b9b276f2
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Sep03_17-51-29_COE-CS-sv004/events.out.tfevents.1725386201.COE-CS-sv004.3435717.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1636c3fc38c9e1749da12137cab66d3ce236fe57d8fa00893b0412c2d7aac10b
+size 45198

runs/Sep05_22-10-21_COE-CS-sv003/events.out.tfevents.1725574415.COE-CS-sv003.16667.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1c341d64ae8eecb7c4a9027e665b213107f0c0a590a9536fe01f58c6c7fc25
+size 5574

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9988193624557261,
-    "total_flos": 1701364321222656.0,
+    "epoch": 0.9997315436241611,
+    "total_flos": 194880114524160.0,
     "train_loss": 0.0,
-    "train_runtime": 0.0113,
-    "train_samples": 90047,
-    "train_samples_per_second": 2398604.87,
-    "train_steps_per_second": 37460.213
+    "train_runtime": 0.0085,
+    "train_samples": 98952,
+    "train_samples_per_second": 3497894.361,
+    "train_steps_per_second": 109279.854
 }

trainer_state.json

@@ -1,628 +1,1342 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9988193624557261,
+  "epoch": 0.9997315436241611,
   "eval_steps": 500,
-  "global_step": 423,
+  "global_step": 931,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0023612750885478157,
-      "grad_norm": 1.5131582373299473,
-      "learning_rate": 2.3255813953488374e-07,
-      "loss": 1.3255,
+      "epoch": 0.0010738255033557046,
+      "grad_norm": 22.873438223486826,
+      "learning_rate": 1.0638297872340426e-07,
+      "loss": 1.3013,
       "step": 1
     },
     {
-      "epoch": 0.011806375442739079,
-      "grad_norm": 1.4970470206944162,
-      "learning_rate": 1.1627906976744188e-06,
-      "loss": 1.3229,
+      "epoch": 0.005369127516778523,
+      "grad_norm": 21.159527745955735,
+      "learning_rate": 5.319148936170213e-07,
+      "loss": 1.3281,
       "step": 5
     },
     {
-      "epoch": 0.023612750885478158,
-      "grad_norm": 1.4534560068903173,
-      "learning_rate": 2.3255813953488376e-06,
-      "loss": 1.3249,
+      "epoch": 0.010738255033557046,
+      "grad_norm": 8.15243855646548,
+      "learning_rate": 1.0638297872340427e-06,
+      "loss": 1.1934,
       "step": 10
     },
     {
-      "epoch": 0.03541912632821724,
-      "grad_norm": 1.3529390744329275,
-      "learning_rate": 3.4883720930232564e-06,
-      "loss": 1.3105,
+      "epoch": 0.016107382550335572,
+      "grad_norm": 7.876837115931939,
+      "learning_rate": 1.595744680851064e-06,
+      "loss": 1.0403,
       "step": 15
     },
     {
-      "epoch": 0.047225501770956316,
-      "grad_norm": 1.1377914835607748,
-      "learning_rate": 4.651162790697675e-06,
-      "loss": 1.2795,
+      "epoch": 0.021476510067114093,
+      "grad_norm": 3.0351073481295128,
+      "learning_rate": 2.1276595744680853e-06,
+      "loss": 0.9152,
       "step": 20
     },
     {
-      "epoch": 0.0590318772136954,
-      "grad_norm": 0.9300447218970413,
-      "learning_rate": 5.8139534883720935e-06,
-      "loss": 1.2542,
+      "epoch": 0.026845637583892617,
+      "grad_norm": 2.4261598045634685,
+      "learning_rate": 2.6595744680851065e-06,
+      "loss": 0.8749,
       "step": 25
     },
     {
-      "epoch": 0.07083825265643448,
-      "grad_norm": 0.7033484136291787,
-      "learning_rate": 6.976744186046513e-06,
-      "loss": 1.2254,
+      "epoch": 0.032214765100671144,
+      "grad_norm": 2.216819042623432,
+      "learning_rate": 3.191489361702128e-06,
+      "loss": 0.8366,
       "step": 30
     },
     {
-      "epoch": 0.08264462809917356,
-      "grad_norm": 0.5519511550883764,
-      "learning_rate": 8.139534883720931e-06,
-      "loss": 1.1956,
+      "epoch": 0.03758389261744966,
+      "grad_norm": 2.282450409349314,
+      "learning_rate": 3.723404255319149e-06,
+      "loss": 0.818,
       "step": 35
     },
     {
-      "epoch": 0.09445100354191263,
-      "grad_norm": 0.530461549075621,
-      "learning_rate": 9.30232558139535e-06,
-      "loss": 1.1609,
+      "epoch": 0.042953020134228186,
+      "grad_norm": 2.2660596456862643,
+      "learning_rate": 4.255319148936171e-06,
+      "loss": 0.7985,
       "step": 40
     },
     {
-      "epoch": 0.10625737898465171,
-      "grad_norm": 0.4882723618310303,
-      "learning_rate": 9.999316524962347e-06,
-      "loss": 1.1353,
+      "epoch": 0.04832214765100671,
+      "grad_norm": 2.367380807091964,
+      "learning_rate": 4.787234042553192e-06,
+      "loss": 0.7886,
       "step": 45
     },
     {
-      "epoch": 0.1180637544273908,
-      "grad_norm": 0.43070884824119315,
-      "learning_rate": 9.991629576543164e-06,
-      "loss": 1.1108,
+      "epoch": 0.053691275167785234,
+      "grad_norm": 2.385525049214499,
+      "learning_rate": 5.319148936170213e-06,
+      "loss": 0.7777,
       "step": 50
     },
     {
-      "epoch": 0.12987012987012986,
-      "grad_norm": 0.4326223460490506,
-      "learning_rate": 9.975414512725058e-06,
-      "loss": 1.0912,
+      "epoch": 0.05906040268456376,
+      "grad_norm": 2.5856947730866446,
+      "learning_rate": 5.851063829787235e-06,
+      "loss": 0.7546,
       "step": 55
     },
     {
-      "epoch": 0.14167650531286896,
-      "grad_norm": 0.3990410735697587,
-      "learning_rate": 9.95069903667256e-06,
-      "loss": 1.0684,
+      "epoch": 0.06442953020134229,
+      "grad_norm": 2.4085256812489084,
+      "learning_rate": 6.382978723404256e-06,
+      "loss": 0.7524,
       "step": 60
     },
     {
-      "epoch": 0.15348288075560804,
-      "grad_norm": 0.4102324061429566,
-      "learning_rate": 9.917525374361913e-06,
-      "loss": 1.0513,
+      "epoch": 0.0697986577181208,
+      "grad_norm": 2.4401920680003157,
+      "learning_rate": 6.914893617021278e-06,
+      "loss": 0.7291,
       "step": 65
     },
     {
-      "epoch": 0.1652892561983471,
-      "grad_norm": 0.4553698218636962,
-      "learning_rate": 9.8759502024387e-06,
-      "loss": 1.0205,
+      "epoch": 0.07516778523489932,
+      "grad_norm": 2.3253305278745215,
+      "learning_rate": 7.446808510638298e-06,
+      "loss": 0.7272,
       "step": 70
     },
     {
-      "epoch": 0.1770956316410862,
-      "grad_norm": 0.4487232032461124,
-      "learning_rate": 9.826044551386743e-06,
-      "loss": 0.9933,
+      "epoch": 0.08053691275167785,
+      "grad_norm": 2.45450168917715,
+      "learning_rate": 7.97872340425532e-06,
+      "loss": 0.7127,
       "step": 75
     },
     {
-      "epoch": 0.18890200708382526,
-      "grad_norm": 0.37710863024094676,
-      "learning_rate": 9.767893684173722e-06,
-      "loss": 0.9569,
+      "epoch": 0.08590604026845637,
+      "grad_norm": 2.127447895073466,
+      "learning_rate": 8.510638297872341e-06,
+      "loss": 0.7041,
       "step": 80
     },
     {
-      "epoch": 0.20070838252656434,
-      "grad_norm": 0.29780121644785873,
-      "learning_rate": 9.701596950580807e-06,
-      "loss": 0.944,
+      "epoch": 0.0912751677852349,
+      "grad_norm": 2.3028444099891705,
+      "learning_rate": 9.042553191489362e-06,
+      "loss": 0.7037,
       "step": 85
     },
     {
-      "epoch": 0.21251475796930341,
-      "grad_norm": 0.2779699178964329,
-      "learning_rate": 9.627267617465243e-06,
-      "loss": 0.9348,
+      "epoch": 0.09664429530201342,
+      "grad_norm": 2.5309118167527616,
+      "learning_rate": 9.574468085106385e-06,
+      "loss": 0.6988,
       "step": 90
     },
     {
-      "epoch": 0.2243211334120425,
-      "grad_norm": 0.25856375372275114,
-      "learning_rate": 9.545032675245814e-06,
-      "loss": 0.931,
+      "epoch": 0.10201342281879194,
+      "grad_norm": 2.4219930767340743,
+      "learning_rate": 9.999964780082996e-06,
+      "loss": 0.6903,
       "step": 95
     },
     {
-      "epoch": 0.2361275088547816,
-      "grad_norm": 0.2613892946414014,
-      "learning_rate": 9.45503262094184e-06,
-      "loss": 0.9177,
+      "epoch": 0.10738255033557047,
+      "grad_norm": 2.111757405385662,
+      "learning_rate": 9.998732135085665e-06,
+      "loss": 0.6801,
       "step": 100
     },
     {
-      "epoch": 0.24793388429752067,
-      "grad_norm": 0.2354999119122035,
-      "learning_rate": 9.357421218136387e-06,
-      "loss": 0.9148,
+      "epoch": 0.11275167785234899,
+      "grad_norm": 2.2202581015134077,
+      "learning_rate": 9.995738990383743e-06,
+      "loss": 0.6907,
       "step": 105
     },
     {
-      "epoch": 0.2597402597402597,
-      "grad_norm": 0.2162870999711504,
-      "learning_rate": 9.252365234273754e-06,
-      "loss": 0.9164,
+      "epoch": 0.11812080536912752,
+      "grad_norm": 2.626765418959801,
+      "learning_rate": 9.990986400130607e-06,
+      "loss": 0.686,
       "step": 110
     },
     {
-      "epoch": 0.2715466351829988,
-      "grad_norm": 0.22180996969883549,
-      "learning_rate": 9.140044155740102e-06,
-      "loss": 0.9011,
+      "epoch": 0.12348993288590604,
+      "grad_norm": 2.2905312730653424,
+      "learning_rate": 9.984476038137437e-06,
+      "loss": 0.6687,
       "step": 115
     },
     {
-      "epoch": 0.2833530106257379,
-      "grad_norm": 0.22437103522632068,
-      "learning_rate": 9.02064988121396e-06,
-      "loss": 0.9006,
+      "epoch": 0.12885906040268458,
+      "grad_norm": 2.190663676539029,
+      "learning_rate": 9.97621019728372e-06,
+      "loss": 0.6776,
       "step": 120
     },
     {
-      "epoch": 0.29515938606847697,
-      "grad_norm": 0.21463636431913852,
-      "learning_rate": 8.894386393810563e-06,
-      "loss": 0.8936,
+      "epoch": 0.1342281879194631,
+      "grad_norm": 2.2515596629136825,
+      "learning_rate": 9.966191788709716e-06,
+      "loss": 0.6694,
       "step": 125
     },
     {
-      "epoch": 0.3069657615112161,
-      "grad_norm": 0.21659671484135407,
-      "learning_rate": 8.761469412580126e-06,
-      "loss": 0.892,
+      "epoch": 0.1395973154362416,
+      "grad_norm": 2.168375563832784,
+      "learning_rate": 9.954424340791195e-06,
+      "loss": 0.6752,
       "step": 130
     },
     {
-      "epoch": 0.3187721369539551,
-      "grad_norm": 0.22355054167698388,
-      "learning_rate": 8.622126023955446e-06,
-      "loss": 0.892,
+      "epoch": 0.14496644295302014,
+      "grad_norm": 2.088272443654766,
+      "learning_rate": 9.940911997896774e-06,
+      "loss": 0.6661,
       "step": 135
     },
     {
-      "epoch": 0.3305785123966942,
-      "grad_norm": 0.21653458201632308,
-      "learning_rate": 8.476594293778561e-06,
-      "loss": 0.8925,
+      "epoch": 0.15033557046979865,
+      "grad_norm": 2.173874083902431,
+      "learning_rate": 9.925659518928316e-06,
+      "loss": 0.6563,
       "step": 140
     },
     {
-      "epoch": 0.34238488783943327,
-      "grad_norm": 0.222501084583819,
-      "learning_rate": 8.325122860569241e-06,
-      "loss": 0.8864,
+      "epoch": 0.15570469798657718,
+      "grad_norm": 2.0589658736370757,
+      "learning_rate": 9.908672275644898e-06,
+      "loss": 0.6607,
       "step": 145
     },
     {
-      "epoch": 0.3541912632821724,
-      "grad_norm": 0.21898064700237985,
-      "learning_rate": 8.167970510730254e-06,
-      "loss": 0.8849,
+      "epoch": 0.1610738255033557,
+      "grad_norm": 2.147351094566772,
+      "learning_rate": 9.889956250770933e-06,
+      "loss": 0.6575,
       "step": 150
     },
     {
-      "epoch": 0.3659976387249115,
-      "grad_norm": 0.22115057529251225,
-      "learning_rate": 8.005405736415127e-06,
-      "loss": 0.8815,
+      "epoch": 0.16644295302013423,
+      "grad_norm": 2.3349226051109584,
+      "learning_rate": 9.86951803588912e-06,
+      "loss": 0.6527,
       "step": 155
     },
     {
-      "epoch": 0.3778040141676505,
-      "grad_norm": 0.21778385690198585,
-      "learning_rate": 7.837706276813819e-06,
-      "loss": 0.8806,
+      "epoch": 0.17181208053691274,
+      "grad_norm": 2.281471579219723,
+      "learning_rate": 9.847364829118963e-06,
+      "loss": 0.6544,
       "step": 160
     },
     {
-      "epoch": 0.38961038961038963,
-      "grad_norm": 0.2293476838871522,
-      "learning_rate": 7.66515864363997e-06,
-      "loss": 0.8781,
+      "epoch": 0.17718120805369128,
+      "grad_norm": 2.1332428982031715,
+      "learning_rate": 9.82350443258166e-06,
+      "loss": 0.6479,
       "step": 165
     },
     {
-      "epoch": 0.4014167650531287,
-      "grad_norm": 0.23331956197796366,
-      "learning_rate": 7.488057631630438e-06,
-      "loss": 0.8742,
+      "epoch": 0.1825503355704698,
+      "grad_norm": 2.0804446588419045,
+      "learning_rate": 9.797945249652295e-06,
+      "loss": 0.6439,
       "step": 170
     },
     {
-      "epoch": 0.4132231404958678,
-      "grad_norm": 0.21721254042866397,
-      "learning_rate": 7.30670581489344e-06,
-      "loss": 0.8805,
+      "epoch": 0.18791946308724833,
+      "grad_norm": 2.0585417674642636,
+      "learning_rate": 9.770696282000245e-06,
+      "loss": 0.6321,
       "step": 175
     },
     {
-      "epoch": 0.42502951593860683,
-      "grad_norm": 0.23010261349577163,
-      "learning_rate": 7.121413029965769e-06,
-      "loss": 0.8742,
+      "epoch": 0.19328859060402684,
+      "grad_norm": 2.1666332876300762,
+      "learning_rate": 9.741767126418898e-06,
+      "loss": 0.6336,
       "step": 180
     },
     {
-      "epoch": 0.43683589138134593,
-      "grad_norm": 0.2206787567833532,
-      "learning_rate": 6.932495846462262e-06,
-      "loss": 0.876,
+      "epoch": 0.19865771812080538,
+      "grad_norm": 2.0205631171866543,
+      "learning_rate": 9.711167971445766e-06,
+      "loss": 0.6306,
       "step": 185
     },
     {
-      "epoch": 0.448642266824085,
-      "grad_norm": 0.2398992468500764,
-      "learning_rate": 6.7402770262219234e-06,
-      "loss": 0.8703,
+      "epoch": 0.2040268456375839,
+      "grad_norm": 2.4514209883366807,
+      "learning_rate": 9.67890959377418e-06,
+      "loss": 0.624,
       "step": 190
     },
     {
-      "epoch": 0.4604486422668241,
-      "grad_norm": 0.2448460311968614,
-      "learning_rate": 6.545084971874738e-06,
-      "loss": 0.873,
+      "epoch": 0.20939597315436242,
+      "grad_norm": 2.011077236989635,
+      "learning_rate": 9.645003354457872e-06,
+      "loss": 0.6176,
       "step": 195
     },
     {
-      "epoch": 0.4722550177095632,
-      "grad_norm": 0.2612008536174653,
-      "learning_rate": 6.34725316577129e-06,
-      "loss": 0.8746,
+      "epoch": 0.21476510067114093,
+      "grad_norm": 2.157067702163037,
+      "learning_rate": 9.60946119490972e-06,
+      "loss": 0.6231,
       "step": 200
     },
     {
-      "epoch": 0.48406139315230223,
-      "grad_norm": 0.24693316413096483,
-      "learning_rate": 6.147119600233758e-06,
-      "loss": 0.8736,
+      "epoch": 0.22013422818791947,
+      "grad_norm": 1.851876954712529,
+      "learning_rate": 9.57229563269612e-06,
+      "loss": 0.6203,
       "step": 205
     },
     {
-      "epoch": 0.49586776859504134,
-      "grad_norm": 0.24534026805051296,
-      "learning_rate": 5.945026200101702e-06,
-      "loss": 0.8652,
+      "epoch": 0.22550335570469798,
+      "grad_norm": 1.9636046456639136,
+      "learning_rate": 9.533519757128426e-06,
+      "loss": 0.6186,
       "step": 210
     },
     {
-      "epoch": 0.5076741440377804,
-      "grad_norm": 0.23005356270231522,
-      "learning_rate": 5.74131823855921e-06,
-      "loss": 0.8624,
+      "epoch": 0.23087248322147652,
+      "grad_norm": 2.374396960285375,
+      "learning_rate": 9.49314722465304e-06,
+      "loss": 0.6157,
       "step": 215
     },
     {
-      "epoch": 0.5194805194805194,
-      "grad_norm": 0.24466967197626655,
-      "learning_rate": 5.53634374724146e-06,
-      "loss": 0.8639,
+      "epoch": 0.23624161073825503,
+      "grad_norm": 2.072084337879624,
+      "learning_rate": 9.451192254041759e-06,
+      "loss": 0.6157,
       "step": 220
     },
     {
-      "epoch": 0.5312868949232585,
-      "grad_norm": 0.24160067474761546,
-      "learning_rate": 5.3304529216284974e-06,
-      "loss": 0.8632,
+      "epoch": 0.24161073825503357,
+      "grad_norm": 2.0213326681347605,
+      "learning_rate": 9.407669621384073e-06,
+      "loss": 0.619,
       "step": 225
     },
     {
-      "epoch": 0.5430932703659976,
-      "grad_norm": 0.26220069503373417,
-      "learning_rate": 5.123997522742151e-06,
-      "loss": 0.8639,
+      "epoch": 0.24697986577181208,
+      "grad_norm": 2.05541918839593,
+      "learning_rate": 9.362594654883185e-06,
+      "loss": 0.6043,
       "step": 230
     },
     {
-      "epoch": 0.5548996458087367,
-      "grad_norm": 0.264066001522196,
-      "learning_rate": 4.917330276168208e-06,
-      "loss": 0.8623,
+      "epoch": 0.2523489932885906,
+      "grad_norm": 1.9647313060735423,
+      "learning_rate": 9.31598322945759e-06,
+      "loss": 0.5877,
       "step": 235
     },
     {
-      "epoch": 0.5667060212514758,
-      "grad_norm": 0.24428345928123874,
-      "learning_rate": 4.710804269430681e-06,
-      "loss": 0.8692,
+      "epoch": 0.25771812080536916,
+      "grad_norm": 2.321446787508396,
+      "learning_rate": 9.267851761150092e-06,
+      "loss": 0.6038,
       "step": 240
     },
     {
-      "epoch": 0.5785123966942148,
-      "grad_norm": 0.24362157236851636,
-      "learning_rate": 4.504772348747687e-06,
-      "loss": 0.8617,
+      "epoch": 0.26308724832214764,
+      "grad_norm": 1.954968634006319,
+      "learning_rate": 9.218217201346251e-06,
+      "loss": 0.5973,
       "step": 245
     },
     {
-      "epoch": 0.5903187721369539,
-      "grad_norm": 0.2605786039459921,
-      "learning_rate": 4.299586516199611e-06,
-      "loss": 0.8631,
+      "epoch": 0.2684563758389262,
+      "grad_norm": 1.9588200511779104,
+      "learning_rate": 9.167097030804289e-06,
+      "loss": 0.5952,
       "step": 250
     },
     {
-      "epoch": 0.602125147579693,
-      "grad_norm": 0.24469411057849033,
-      "learning_rate": 4.0955973283394525e-06,
-      "loss": 0.8519,
+      "epoch": 0.2738255033557047,
+      "grad_norm": 1.978200194451611,
+      "learning_rate": 9.114509253498554e-06,
+      "loss": 0.5956,
       "step": 255
     },
     {
-      "epoch": 0.6139315230224321,
-      "grad_norm": 0.24265909990188317,
-      "learning_rate": 3.893153297272829e-06,
-      "loss": 0.858,
+      "epoch": 0.2791946308724832,
+      "grad_norm": 1.9082868769634187,
+      "learning_rate": 9.060472390278717e-06,
+      "loss": 0.5834,
       "step": 260
     },
     {
-      "epoch": 0.6257378984651711,
-      "grad_norm": 0.2431981393362141,
-      "learning_rate": 3.6926002952309015e-06,
-      "loss": 0.8574,
+      "epoch": 0.28456375838926173,
+      "grad_norm": 1.9947479259010255,
+      "learning_rate": 9.005005472346923e-06,
+      "loss": 0.5878,
       "step": 265
     },
     {
-      "epoch": 0.6375442739079102,
-      "grad_norm": 0.25137244252341084,
-      "learning_rate": 3.4942809636534637e-06,
-      "loss": 0.8584,
+      "epoch": 0.28993288590604027,
+      "grad_norm": 2.0101949394033443,
+      "learning_rate": 8.948128034555212e-06,
+      "loss": 0.5815,
       "step": 270
     },
     {
-      "epoch": 0.6493506493506493,
-      "grad_norm": 0.2520911165703089,
-      "learning_rate": 3.298534127791785e-06,
-      "loss": 0.8578,
+      "epoch": 0.2953020134228188,
+      "grad_norm": 1.855173856877532,
+      "learning_rate": 8.889860108525544e-06,
+      "loss": 0.5749,
       "step": 275
     },
     {
-      "epoch": 0.6611570247933884,
-      "grad_norm": 0.2651721936087408,
-      "learning_rate": 3.105694217831361e-06,
-      "loss": 0.8589,
+      "epoch": 0.3006711409395973,
+      "grad_norm": 1.8947595029633266,
+      "learning_rate": 8.83022221559489e-06,
+      "loss": 0.5597,
       "step": 280
     },
     {
-      "epoch": 0.6729634002361276,
-      "grad_norm": 0.2546800394311146,
-      "learning_rate": 2.9160906975235493e-06,
-      "loss": 0.8523,
+      "epoch": 0.30604026845637583,
+      "grad_norm": 2.0225835192845825,
+      "learning_rate": 8.76923535958783e-06,
+      "loss": 0.5683,
       "step": 285
     },
     {
-      "epoch": 0.6847697756788665,
-      "grad_norm": 0.25316819030742643,
-      "learning_rate": 2.7300475013022666e-06,
-      "loss": 0.8552,
+      "epoch": 0.31140939597315437,
+      "grad_norm": 2.047769165835403,
+      "learning_rate": 8.706921019419237e-06,
+      "loss": 0.5693,
       "step": 290
     },
     {
-      "epoch": 0.6965761511216056,
-      "grad_norm": 0.2509477836308168,
-      "learning_rate": 2.5478824808474613e-06,
-      "loss": 0.8574,
+      "epoch": 0.3167785234899329,
+      "grad_norm": 1.996841996415469,
+      "learning_rate": 8.643301141529619e-06,
+      "loss": 0.576,
       "step": 295
     },
     {
-      "epoch": 0.7083825265643447,
-      "grad_norm": 0.22985580266469613,
-      "learning_rate": 2.3699068620408305e-06,
-      "loss": 0.8525,
+      "epoch": 0.3221476510067114,
+      "grad_norm": 2.085317513003649,
+      "learning_rate": 8.578398132155846e-06,
+      "loss": 0.5586,
       "step": 300
     },
     {
-      "epoch": 0.7201889020070839,
-      "grad_norm": 0.24633435787531885,
-      "learning_rate": 2.1964247132416373e-06,
-      "loss": 0.8541,
+      "epoch": 0.3275167785234899,
+      "grad_norm": 2.2696870964241342,
+      "learning_rate": 8.512234849439887e-06,
+      "loss": 0.5487,
       "step": 305
     },
     {
-      "epoch": 0.731995277449823,
-      "grad_norm": 0.2550223994617769,
-      "learning_rate": 2.027732425791011e-06,
-      "loss": 0.8575,
+      "epoch": 0.33288590604026846,
+      "grad_norm": 2.295292164613049,
+      "learning_rate": 8.444834595378434e-06,
+      "loss": 0.5657,
       "step": 310
     },
     {
-      "epoch": 0.743801652892562,
-      "grad_norm": 0.250598594634277,
-      "learning_rate": 1.864118207632315e-06,
-      "loss": 0.8584,
+      "epoch": 0.338255033557047,
+      "grad_norm": 1.9535177447525518,
+      "learning_rate": 8.376221107616187e-06,
+      "loss": 0.5492,
       "step": 315
     },
     {
-      "epoch": 0.755608028335301,
-      "grad_norm": 0.2547611444118788,
-      "learning_rate": 1.7058615909127102e-06,
-      "loss": 0.8578,
+      "epoch": 0.3436241610738255,
+      "grad_norm": 1.935697674348923,
+      "learning_rate": 8.306418551085707e-06,
+      "loss": 0.5502,
       "step": 320
     },
     {
-      "epoch": 0.7674144037780402,
-      "grad_norm": 0.2551344265566289,
-      "learning_rate": 1.5532329544071712e-06,
-      "loss": 0.8568,
+      "epoch": 0.348993288590604,
+      "grad_norm": 1.912637650045647,
+      "learning_rate": 8.23545150949679e-06,
+      "loss": 0.5659,
       "step": 325
     },
     {
-      "epoch": 0.7792207792207793,
-      "grad_norm": 0.24284035631227585,
-      "learning_rate": 1.406493061580881e-06,
-      "loss": 0.854,
+      "epoch": 0.35436241610738256,
+      "grad_norm": 2.0813900852367437,
+      "learning_rate": 8.163344976678342e-06,
+      "loss": 0.5512,
       "step": 330
     },
     {
-      "epoch": 0.7910271546635183,
-      "grad_norm": 0.2454427828284572,
-      "learning_rate": 1.2658926150792321e-06,
-      "loss": 0.8543,
+      "epoch": 0.3597315436241611,
+      "grad_norm": 1.9178576647529397,
+      "learning_rate": 8.090124347775837e-06,
+      "loss": 0.5399,
       "step": 335
     },
     {
-      "epoch": 0.8028335301062574,
-      "grad_norm": 0.24171894401645216,
-      "learning_rate": 1.1316718284065536e-06,
-      "loss": 0.8601,
+      "epoch": 0.3651006711409396,
+      "grad_norm": 1.9921209746990098,
+      "learning_rate": 8.0158154103074e-06,
+      "loss": 0.5335,
       "step": 340
     },
     {
-      "epoch": 0.8146399055489965,
-      "grad_norm": 0.24890059165826273,
-      "learning_rate": 1.0040600155253766e-06,
-      "loss": 0.8588,
+      "epoch": 0.3704697986577181,
+      "grad_norm": 2.0251179740048686,
+      "learning_rate": 7.940444335081733e-06,
+      "loss": 0.5412,
       "step": 345
     },
     {
-      "epoch": 0.8264462809917356,
-      "grad_norm": 0.24128541650834764,
-      "learning_rate": 8.832751990773714e-07,
-      "loss": 0.8547,
+      "epoch": 0.37583892617449666,
+      "grad_norm": 2.0237480393180536,
+      "learning_rate": 7.864037666981037e-06,
+      "loss": 0.5318,
       "step": 350
     },
     {
-      "epoch": 0.8382526564344747,
-      "grad_norm": 0.25646382446616656,
-      "learning_rate": 7.695237378953224e-07,
-      "loss": 0.859,
+      "epoch": 0.3812080536912752,
+      "grad_norm": 2.062903496053255,
+      "learning_rate": 7.786622315612182e-06,
+      "loss": 0.5391,
       "step": 355
     },
     {
-      "epoch": 0.8500590318772137,
-      "grad_norm": 0.2460481416324142,
-      "learning_rate": 6.629999744425236e-07,
-      "loss": 0.8567,
+      "epoch": 0.3865771812080537,
+      "grad_norm": 1.931828289441635,
+      "learning_rate": 7.708225545829446e-06,
+      "loss": 0.525,
       "step": 360
     },
     {
-      "epoch": 0.8618654073199528,
-      "grad_norm": 0.24513983983677975,
-      "learning_rate": 5.63885902781941e-07,
-      "loss": 0.8596,
+      "epoch": 0.3919463087248322,
+      "grad_norm": 2.004972595205748,
+      "learning_rate": 7.6288749681321115e-06,
+      "loss": 0.5173,
       "step": 365
     },
     {
-      "epoch": 0.8736717827626919,
-      "grad_norm": 0.2343335791217957,
-      "learning_rate": 4.723508576424063e-07,
-      "loss": 0.8552,
+      "epoch": 0.39731543624161075,
+      "grad_norm": 2.0462367199517937,
+      "learning_rate": 7.548598528940354e-06,
+      "loss": 0.5154,
       "step": 370
     },
     {
-      "epoch": 0.885478158205431,
-      "grad_norm": 0.2519214484705661,
-      "learning_rate": 3.885512251130763e-07,
-      "loss": 0.8527,
+      "epoch": 0.40268456375838924,
+      "grad_norm": 2.3727598847724636,
+      "learning_rate": 7.4674245007528135e-06,
+      "loss": 0.5279,
       "step": 375
     },
     {
-      "epoch": 0.89728453364817,
-      "grad_norm": 0.2440125511079392,
-      "learning_rate": 3.126301754604233e-07,
-      "loss": 0.8548,
+      "epoch": 0.4080536912751678,
+      "grad_norm": 1.965518745162171,
+      "learning_rate": 7.385381472189321e-06,
+      "loss": 0.5255,
       "step": 380
     },
     {
-      "epoch": 0.9090909090909091,
-      "grad_norm": 0.2532464399771227,
-      "learning_rate": 2.447174185242324e-07,
-      "loss": 0.854,
+      "epoch": 0.4134228187919463,
+      "grad_norm": 2.0728393681212225,
+      "learning_rate": 7.302498337922293e-06,
+      "loss": 0.5152,
       "step": 385
     },
     {
-      "epoch": 0.9208972845336482,
-      "grad_norm": 0.23622376914066623,
-      "learning_rate": 1.849289821105199e-07,
-      "loss": 0.8521,
+      "epoch": 0.41879194630872485,
+      "grad_norm": 2.0074609174878906,
+      "learning_rate": 7.218804288500343e-06,
+      "loss": 0.5224,
       "step": 390
     },
     {
-      "epoch": 0.9327036599763873,
-      "grad_norm": 0.2444798272238538,
-      "learning_rate": 1.333670137599713e-07,
-      "loss": 0.8532,
+      "epoch": 0.42416107382550333,
+      "grad_norm": 1.9472600550525063,
+      "learning_rate": 7.134328800067684e-06,
+      "loss": 0.5209,
       "step": 395
     },
     {
-      "epoch": 0.9445100354191264,
-      "grad_norm": 0.23429527809735437,
-      "learning_rate": 9.011960623058202e-08,
-      "loss": 0.8532,
+      "epoch": 0.42953020134228187,
+      "grad_norm": 1.9949043902713617,
+      "learning_rate": 7.049101623982938e-06,
+      "loss": 0.5082,
       "step": 400
     },
     {
-      "epoch": 0.9563164108618654,
-      "grad_norm": 0.23623057232372727,
-      "learning_rate": 5.526064699265754e-08,
-      "loss": 0.8506,
+      "epoch": 0.4348993288590604,
+      "grad_norm": 1.834654212769176,
+      "learning_rate": 6.963152776341044e-06,
+      "loss": 0.5149,
       "step": 405
     },
     {
-      "epoch": 0.9681227863046045,
-      "grad_norm": 0.2482774375940229,
-      "learning_rate": 2.884969199331178e-08,
-      "loss": 0.8587,
+      "epoch": 0.44026845637583895,
+      "grad_norm": 2.0502871429630822,
+      "learning_rate": 6.876512527401897e-06,
+      "loss": 0.5131,
       "step": 410
     },
     {
-      "epoch": 0.9799291617473436,
-      "grad_norm": 0.23526019800815218,
-      "learning_rate": 1.0931863906127327e-08,
-      "loss": 0.8558,
+      "epoch": 0.44563758389261743,
+      "grad_norm": 2.000365576779467,
+      "learning_rate": 6.789211390929497e-06,
+      "loss": 0.5016,
       "step": 415
     },
     {
-      "epoch": 0.9917355371900827,
-      "grad_norm": 0.24074172289610016,
-      "learning_rate": 1.53777503982655e-09,
-      "loss": 0.853,
+      "epoch": 0.45100671140939597,
+      "grad_norm": 2.0157623562806353,
+      "learning_rate": 6.701280113445324e-06,
+      "loss": 0.4958,
       "step": 420
     },
     {
-      "epoch": 0.9988193624557261,
-      "eval_loss": 0.8673050403594971,
-      "eval_runtime": 1.109,
-      "eval_samples_per_second": 1.804,
-      "eval_steps_per_second": 0.902,
-      "step": 423
+      "epoch": 0.4563758389261745,
+      "grad_norm": 1.970116584111867,
+      "learning_rate": 6.6127496633997475e-06,
+      "loss": 0.5003,
+      "step": 425
     },
     {
-      "epoch": 0.9988193624557261,
-      "step": 423,
-      "total_flos": 1701364321222656.0,
+      "epoch": 0.46174496644295304,
+      "grad_norm": 1.9483867090820592,
+      "learning_rate": 6.523651220265269e-06,
+      "loss": 0.4942,
+      "step": 430
+    },
+    {
+      "epoch": 0.4671140939597315,
+      "grad_norm": 2.099467823611523,
+      "learning_rate": 6.434016163555452e-06,
+      "loss": 0.4952,
+      "step": 435
+    },
+    {
+      "epoch": 0.47248322147651006,
+      "grad_norm": 1.9558069283541033,
+      "learning_rate": 6.343876061773385e-06,
+      "loss": 0.4924,
+      "step": 440
+    },
+    {
+      "epoch": 0.4778523489932886,
+      "grad_norm": 1.9847822609941113,
+      "learning_rate": 6.2532626612936035e-06,
+      "loss": 0.4823,
+      "step": 445
+    },
+    {
+      "epoch": 0.48322147651006714,
+      "grad_norm": 2.0688837915005993,
+      "learning_rate": 6.162207875181354e-06,
+      "loss": 0.4913,
+      "step": 450
+    },
+    {
+      "epoch": 0.4885906040268456,
+      "grad_norm": 1.9342878293544008,
+      "learning_rate": 6.070743771953157e-06,
+      "loss": 0.4769,
+      "step": 455
+    },
+    {
+      "epoch": 0.49395973154362416,
+      "grad_norm": 2.043215000887027,
+      "learning_rate": 5.978902564282616e-06,
+      "loss": 0.4752,
+      "step": 460
+    },
+    {
+      "epoch": 0.4993288590604027,
+      "grad_norm": 1.8676181620631764,
+      "learning_rate": 5.886716597655472e-06,
+      "loss": 0.4701,
+      "step": 465
+    },
+    {
+      "epoch": 0.5046979865771812,
+      "grad_norm": 1.9595541482385936,
+      "learning_rate": 5.7942183389778536e-06,
+      "loss": 0.4696,
+      "step": 470
+    },
+    {
+      "epoch": 0.5100671140939598,
+      "grad_norm": 1.825167080567174,
+      "learning_rate": 5.701440365141799e-06,
+      "loss": 0.4625,
+      "step": 475
+    },
+    {
+      "epoch": 0.5154362416107383,
+      "grad_norm": 1.9786129201297613,
+      "learning_rate": 5.608415351552014e-06,
+      "loss": 0.4596,
+      "step": 480
+    },
+    {
+      "epoch": 0.5208053691275167,
+      "grad_norm": 1.9920386733808053,
+      "learning_rate": 5.515176060617945e-06,
+      "loss": 0.4684,
+      "step": 485
+    },
+    {
+      "epoch": 0.5261744966442953,
+      "grad_norm": 1.9116799488281817,
+      "learning_rate": 5.421755330215223e-06,
+      "loss": 0.4659,
+      "step": 490
+    },
+    {
+      "epoch": 0.5315436241610738,
+      "grad_norm": 1.948129234992053,
+      "learning_rate": 5.328186062120509e-06,
+      "loss": 0.4436,
+      "step": 495
+    },
+    {
+      "epoch": 0.5369127516778524,
+      "grad_norm": 2.078748185289108,
+      "learning_rate": 5.23450121042383e-06,
+      "loss": 0.4548,
+      "step": 500
+    },
+    {
+      "epoch": 0.5422818791946309,
+      "grad_norm": 1.9444657978297817,
+      "learning_rate": 5.140733769922525e-06,
+      "loss": 0.4533,
+      "step": 505
+    },
+    {
+      "epoch": 0.5476510067114094,
+      "grad_norm": 1.9186109778492548,
+      "learning_rate": 5.0469167645008245e-06,
+      "loss": 0.4484,
+      "step": 510
+    },
+    {
+      "epoch": 0.553020134228188,
+      "grad_norm": 2.115910772434904,
+      "learning_rate": 4.953083235499177e-06,
+      "loss": 0.4557,
+      "step": 515
+    },
+    {
+      "epoch": 0.5583892617449664,
+      "grad_norm": 1.9469832075906568,
+      "learning_rate": 4.859266230077474e-06,
+      "loss": 0.4439,
+      "step": 520
+    },
+    {
+      "epoch": 0.5637583892617449,
+      "grad_norm": 1.90470271131317,
+      "learning_rate": 4.7654987895761705e-06,
+      "loss": 0.4431,
+      "step": 525
+    },
+    {
+      "epoch": 0.5691275167785235,
+      "grad_norm": 1.941709380965833,
+      "learning_rate": 4.671813937879494e-06,
+      "loss": 0.4517,
+      "step": 530
+    },
+    {
+      "epoch": 0.574496644295302,
+      "grad_norm": 1.9633282455666656,
+      "learning_rate": 4.5782446697847775e-06,
+      "loss": 0.4469,
+      "step": 535
+    },
+    {
+      "epoch": 0.5798657718120805,
+      "grad_norm": 1.8845576495595482,
+      "learning_rate": 4.484823939382056e-06,
+      "loss": 0.4316,
+      "step": 540
+    },
+    {
+      "epoch": 0.5852348993288591,
+      "grad_norm": 1.8811789086409214,
+      "learning_rate": 4.391584648447989e-06,
+      "loss": 0.4423,
+      "step": 545
+    },
+    {
+      "epoch": 0.5906040268456376,
+      "grad_norm": 1.9555309537231733,
+      "learning_rate": 4.298559634858202e-06,
+      "loss": 0.4368,
+      "step": 550
+    },
+    {
+      "epoch": 0.5959731543624162,
+      "grad_norm": 2.0555513727023578,
+      "learning_rate": 4.205781661022146e-06,
+      "loss": 0.4298,
+      "step": 555
+    },
+    {
+      "epoch": 0.6013422818791946,
+      "grad_norm": 1.8777742161685087,
+      "learning_rate": 4.1132834023445304e-06,
+      "loss": 0.426,
+      "step": 560
+    },
+    {
+      "epoch": 0.6067114093959731,
+      "grad_norm": 1.939266556920773,
+      "learning_rate": 4.021097435717386e-06,
+      "loss": 0.4216,
+      "step": 565
+    },
+    {
+      "epoch": 0.6120805369127517,
+      "grad_norm": 1.9261147196325419,
+      "learning_rate": 3.929256228046845e-06,
+      "loss": 0.4285,
+      "step": 570
+    },
+    {
+      "epoch": 0.6174496644295302,
+      "grad_norm": 1.805107172046572,
+      "learning_rate": 3.837792124818647e-06,
+      "loss": 0.416,
+      "step": 575
+    },
+    {
+      "epoch": 0.6228187919463087,
+      "grad_norm": 1.8704231334410426,
+      "learning_rate": 3.7467373387063973e-06,
+      "loss": 0.4191,
+      "step": 580
+    },
+    {
+      "epoch": 0.6281879194630873,
+      "grad_norm": 1.9935959039830586,
+      "learning_rate": 3.656123938226618e-06,
+      "loss": 0.4183,
+      "step": 585
+    },
+    {
+      "epoch": 0.6335570469798658,
+      "grad_norm": 1.9248727434625008,
+      "learning_rate": 3.5659838364445505e-06,
+      "loss": 0.4239,
+      "step": 590
+    },
+    {
+      "epoch": 0.6389261744966444,
+      "grad_norm": 1.9448194253706668,
+      "learning_rate": 3.476348779734732e-06,
+      "loss": 0.4118,
+      "step": 595
+    },
+    {
+      "epoch": 0.6442953020134228,
+      "grad_norm": 1.8244567974798083,
+      "learning_rate": 3.387250336600254e-06,
+      "loss": 0.4108,
+      "step": 600
+    },
+    {
+      "epoch": 0.6496644295302013,
+      "grad_norm": 1.8766359486725541,
+      "learning_rate": 3.298719886554677e-06,
+      "loss": 0.4094,
+      "step": 605
+    },
+    {
+      "epoch": 0.6550335570469799,
+      "grad_norm": 1.9861783019472632,
+      "learning_rate": 3.2107886090705035e-06,
+      "loss": 0.4024,
+      "step": 610
+    },
+    {
+      "epoch": 0.6604026845637584,
+      "grad_norm": 1.8918824877700715,
+      "learning_rate": 3.1234874725981045e-06,
+      "loss": 0.4086,
+      "step": 615
+    },
+    {
+      "epoch": 0.6657718120805369,
+      "grad_norm": 2.0714978550264003,
+      "learning_rate": 3.036847223658958e-06,
+      "loss": 0.396,
+      "step": 620
+    },
+    {
+      "epoch": 0.6711409395973155,
+      "grad_norm": 1.9560452224763145,
+      "learning_rate": 2.950898376017064e-06,
+      "loss": 0.4035,
+      "step": 625
+    },
+    {
+      "epoch": 0.676510067114094,
+      "grad_norm": 2.043363405835207,
+      "learning_rate": 2.865671199932318e-06,
+      "loss": 0.4068,
+      "step": 630
+    },
+    {
+      "epoch": 0.6818791946308724,
+      "grad_norm": 1.8816775523611597,
+      "learning_rate": 2.781195711499658e-06,
+      "loss": 0.393,
+      "step": 635
+    },
+    {
+      "epoch": 0.687248322147651,
+      "grad_norm": 1.8488910820957973,
+      "learning_rate": 2.697501662077707e-06,
+      "loss": 0.4013,
+      "step": 640
+    },
+    {
+      "epoch": 0.6926174496644295,
+      "grad_norm": 1.862219172494916,
+      "learning_rate": 2.6146185278106807e-06,
+      "loss": 0.3909,
+      "step": 645
+    },
+    {
+      "epoch": 0.697986577181208,
+      "grad_norm": 2.012779150472371,
+      "learning_rate": 2.5325754992471886e-06,
+      "loss": 0.3933,
+      "step": 650
+    },
+    {
+      "epoch": 0.7033557046979866,
+      "grad_norm": 1.925776837174688,
+      "learning_rate": 2.4514014710596467e-06,
+      "loss": 0.3794,
+      "step": 655
+    },
+    {
+      "epoch": 0.7087248322147651,
+      "grad_norm": 1.8323121691067996,
+      "learning_rate": 2.3711250318678906e-06,
+      "loss": 0.3906,
+      "step": 660
+    },
+    {
+      "epoch": 0.7140939597315437,
+      "grad_norm": 1.8449642618290978,
+      "learning_rate": 2.2917744541705544e-06,
+      "loss": 0.3804,
+      "step": 665
+    },
+    {
+      "epoch": 0.7194630872483222,
+      "grad_norm": 1.8385577849219443,
+      "learning_rate": 2.2133776843878185e-06,
+      "loss": 0.3896,
+      "step": 670
+    },
+    {
+      "epoch": 0.7248322147651006,
+      "grad_norm": 1.8863509359096609,
+      "learning_rate": 2.1359623330189655e-06,
+      "loss": 0.3906,
+      "step": 675
+    },
+    {
+      "epoch": 0.7302013422818792,
+      "grad_norm": 1.9412331601540616,
+      "learning_rate": 2.059555664918268e-06,
+      "loss": 0.3872,
+      "step": 680
+    },
+    {
+      "epoch": 0.7355704697986577,
+      "grad_norm": 1.795387975158009,
+      "learning_rate": 1.9841845896926022e-06,
+      "loss": 0.3798,
+      "step": 685
+    },
+    {
+      "epoch": 0.7409395973154362,
+      "grad_norm": 1.843173352636606,
+      "learning_rate": 1.9098756522241634e-06,
+      "loss": 0.3788,
+      "step": 690
+    },
+    {
+      "epoch": 0.7463087248322148,
+      "grad_norm": 1.912174039399666,
+      "learning_rate": 1.8366550233216584e-06,
+      "loss": 0.3695,
+      "step": 695
+    },
+    {
+      "epoch": 0.7516778523489933,
+      "grad_norm": 1.8237677647091424,
+      "learning_rate": 1.7645484905032129e-06,
+      "loss": 0.3933,
+      "step": 700
+    },
+    {
+      "epoch": 0.7570469798657719,
+      "grad_norm": 1.7822160523413584,
+      "learning_rate": 1.6935814489142937e-06,
+      "loss": 0.3781,
+      "step": 705
+    },
+    {
+      "epoch": 0.7624161073825504,
+      "grad_norm": 1.8878271721301074,
+      "learning_rate": 1.6237788923838149e-06,
+      "loss": 0.3751,
+      "step": 710
+    },
+    {
+      "epoch": 0.7677852348993288,
+      "grad_norm": 1.9082160841545055,
+      "learning_rate": 1.555165404621567e-06,
+      "loss": 0.3728,
+      "step": 715
+    },
+    {
+      "epoch": 0.7731543624161074,
+      "grad_norm": 1.816281209572928,
+      "learning_rate": 1.487765150560116e-06,
+      "loss": 0.3777,
+      "step": 720
+    },
+    {
+      "epoch": 0.7785234899328859,
+      "grad_norm": 1.91354765663735,
+      "learning_rate": 1.4216018678441558e-06,
+      "loss": 0.3699,
+      "step": 725
+    },
+    {
+      "epoch": 0.7838926174496644,
+      "grad_norm": 1.9355520287879238,
+      "learning_rate": 1.3566988584703817e-06,
+      "loss": 0.371,
+      "step": 730
+    },
+    {
+      "epoch": 0.789261744966443,
+      "grad_norm": 1.7280419120289419,
+      "learning_rate": 1.293078980580766e-06,
+      "loss": 0.3764,
+      "step": 735
+    },
+    {
+      "epoch": 0.7946308724832215,
+      "grad_norm": 1.9044555736805637,
+      "learning_rate": 1.2307646404121692e-06,
+      "loss": 0.3664,
+      "step": 740
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.891432724737264,
+      "learning_rate": 1.1697777844051105e-06,
+      "loss": 0.3664,
+      "step": 745
+    },
+    {
+      "epoch": 0.8053691275167785,
+      "grad_norm": 1.6814136447042585,
+      "learning_rate": 1.1101398914744565e-06,
+      "loss": 0.3574,
+      "step": 750
+    },
+    {
+      "epoch": 0.810738255033557,
+      "grad_norm": 1.8225153893838018,
+      "learning_rate": 1.0518719654447896e-06,
+      "loss": 0.3684,
+      "step": 755
+    },
+    {
+      "epoch": 0.8161073825503355,
+      "grad_norm": 1.8275415575673986,
+      "learning_rate": 9.949945276530782e-07,
+      "loss": 0.3603,
+      "step": 760
+    },
+    {
+      "epoch": 0.8214765100671141,
+      "grad_norm": 1.853253611406343,
+      "learning_rate": 9.395276097212841e-07,
+      "loss": 0.3651,
+      "step": 765
+    },
+    {
+      "epoch": 0.8268456375838926,
+      "grad_norm": 1.835073747374209,
+      "learning_rate": 8.854907465014479e-07,
+      "loss": 0.3677,
+      "step": 770
+    },
+    {
+      "epoch": 0.8322147651006712,
+      "grad_norm": 1.7695339228605491,
+      "learning_rate": 8.329029691957124e-07,
+      "loss": 0.3606,
+      "step": 775
+    },
+    {
+      "epoch": 0.8375838926174497,
+      "grad_norm": 1.749975183558492,
+      "learning_rate": 7.817827986537508e-07,
+      "loss": 0.3511,
+      "step": 780
+    },
+    {
+      "epoch": 0.8429530201342282,
+      "grad_norm": 1.9426041133261787,
+      "learning_rate": 7.321482388499096e-07,
+      "loss": 0.358,
+      "step": 785
+    },
+    {
+      "epoch": 0.8483221476510067,
+      "grad_norm": 1.8171481735775137,
+      "learning_rate": 6.840167705424106e-07,
+      "loss": 0.3544,
+      "step": 790
+    },
+    {
+      "epoch": 0.8536912751677852,
+      "grad_norm": 1.8412832647944588,
+      "learning_rate": 6.374053451168166e-07,
+      "loss": 0.3661,
+      "step": 795
+    },
+    {
+      "epoch": 0.8590604026845637,
+      "grad_norm": 1.8350930759774366,
+      "learning_rate": 5.92330378615929e-07,
+      "loss": 0.3537,
+      "step": 800
+    },
+    {
+      "epoch": 0.8644295302013423,
+      "grad_norm": 1.723224003576655,
+      "learning_rate": 5.488077459582425e-07,
+      "loss": 0.3553,
+      "step": 805
+    },
+    {
+      "epoch": 0.8697986577181208,
+      "grad_norm": 1.8473917985800616,
+      "learning_rate": 5.068527753469604e-07,
+      "loss": 0.3603,
+      "step": 810
+    },
+    {
+      "epoch": 0.8751677852348994,
+      "grad_norm": 1.8325217665657294,
+      "learning_rate": 4.664802428715753e-07,
+      "loss": 0.348,
+      "step": 815
+    },
+    {
+      "epoch": 0.8805369127516779,
+      "grad_norm": 1.7346304200820148,
+      "learning_rate": 4.2770436730388166e-07,
+      "loss": 0.3498,
+      "step": 820
+    },
+    {
+      "epoch": 0.8859060402684564,
+      "grad_norm": 1.7575151506469913,
+      "learning_rate": 3.9053880509028086e-07,
+      "loss": 0.3616,
+      "step": 825
+    },
+    {
+      "epoch": 0.8912751677852349,
+      "grad_norm": 1.800873655206166,
+      "learning_rate": 3.549966455421305e-07,
+      "loss": 0.3507,
+      "step": 830
+    },
+    {
+      "epoch": 0.8966442953020134,
+      "grad_norm": 1.710054226395714,
+      "learning_rate": 3.2109040622582186e-07,
+      "loss": 0.3527,
+      "step": 835
+    },
+    {
+      "epoch": 0.9020134228187919,
+      "grad_norm": 1.6705083298206793,
+      "learning_rate": 2.8883202855423676e-07,
+      "loss": 0.3516,
+      "step": 840
+    },
+    {
+      "epoch": 0.9073825503355705,
+      "grad_norm": 1.7474504216215188,
+      "learning_rate": 2.582328735811029e-07,
+      "loss": 0.3452,
+      "step": 845
+    },
+    {
+      "epoch": 0.912751677852349,
+      "grad_norm": 1.7463913866112468,
+      "learning_rate": 2.2930371799975593e-07,
+      "loss": 0.3471,
+      "step": 850
+    },
+    {
+      "epoch": 0.9181208053691275,
+      "grad_norm": 1.7297820385087201,
+      "learning_rate": 2.0205475034770606e-07,
+      "loss": 0.3468,
+      "step": 855
+    },
+    {
+      "epoch": 0.9234899328859061,
+      "grad_norm": 1.7482101229826381,
+      "learning_rate": 1.7649556741833995e-07,
+      "loss": 0.3478,
+      "step": 860
+    },
+    {
+      "epoch": 0.9288590604026845,
+      "grad_norm": 1.8150110385873126,
+      "learning_rate": 1.5263517088103862e-07,
+      "loss": 0.3586,
+      "step": 865
+    },
+    {
+      "epoch": 0.934228187919463,
+      "grad_norm": 1.8153438746846808,
+      "learning_rate": 1.304819641108801e-07,
+      "loss": 0.3555,
+      "step": 870
+    },
+    {
+      "epoch": 0.9395973154362416,
+      "grad_norm": 1.8087296470250676,
+      "learning_rate": 1.1004374922906846e-07,
+      "loss": 0.3457,
+      "step": 875
+    },
+    {
+      "epoch": 0.9449664429530201,
+      "grad_norm": 1.7195355098862968,
+      "learning_rate": 9.132772435510362e-08,
+      "loss": 0.349,
+      "step": 880
+    },
+    {
+      "epoch": 0.9503355704697987,
+      "grad_norm": 1.7647364179507874,
+      "learning_rate": 7.434048107168523e-08,
+      "loss": 0.3466,
+      "step": 885
+    },
+    {
+      "epoch": 0.9557046979865772,
+      "grad_norm": 1.9088484806569315,
+      "learning_rate": 5.908800210322696e-08,
+      "loss": 0.3411,
+      "step": 890
+    },
+    {
+      "epoch": 0.9610738255033557,
+      "grad_norm": 1.7271817479075642,
+      "learning_rate": 4.55756592088058e-08,
+      "loss": 0.3484,
+      "step": 895
+    },
+    {
+      "epoch": 0.9664429530201343,
+      "grad_norm": 1.7372584442728163,
+      "learning_rate": 3.3808211290284886e-08,
+      "loss": 0.3433,
+      "step": 900
+    },
+    {
+      "epoch": 0.9718120805369127,
+      "grad_norm": 1.808380468788094,
+      "learning_rate": 2.378980271628195e-08,
+      "loss": 0.3528,
+      "step": 905
+    },
+    {
+      "epoch": 0.9771812080536912,
+      "grad_norm": 1.7605143851131846,
+      "learning_rate": 1.552396186256411e-08,
+      "loss": 0.3445,
+      "step": 910
+    },
+    {
+      "epoch": 0.9825503355704698,
+      "grad_norm": 1.8047618073453124,
+      "learning_rate": 9.013599869394096e-09,
+      "loss": 0.3497,
+      "step": 915
+    },
+    {
+      "epoch": 0.9879194630872483,
+      "grad_norm": 1.7837820069864043,
+      "learning_rate": 4.261009616257638e-09,
+      "loss": 0.3452,
+      "step": 920
+    },
+    {
+      "epoch": 0.9932885906040269,
+      "grad_norm": 1.7898194618346237,
+      "learning_rate": 1.2678649143349485e-09,
+      "loss": 0.3497,
+      "step": 925
+    },
+    {
+      "epoch": 0.9986577181208054,
+      "grad_norm": 1.8036764007244224,
+      "learning_rate": 3.5219917003948003e-11,
+      "loss": 0.346,
+      "step": 930
+    },
+    {
+      "epoch": 0.9997315436241611,
+      "eval_loss": 0.2977491617202759,
+      "eval_runtime": 1.3637,
+      "eval_samples_per_second": 1.467,
+      "eval_steps_per_second": 0.733,
+      "step": 931
+    },
+    {
+      "epoch": 0.9997315436241611,
+      "step": 931,
+      "total_flos": 194880114524160.0,
       "train_loss": 0.0,
-      "train_runtime": 0.0113,
-      "train_samples_per_second": 2398604.87,
-      "train_steps_per_second": 37460.213
+      "train_runtime": 0.0085,
+      "train_samples_per_second": 3497894.361,
+      "train_steps_per_second": 109279.854
     }
   ],
   "logging_steps": 5,
-  "max_steps": 423,
+  "max_steps": 931,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
@@ -638,8 +1352,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 1701364321222656.0,
-  "train_batch_size": 8,
+  "total_flos": 194880114524160.0,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6c12087aca541aaa2c9502933a655e7e92228785f5ba9e1ce617b2a1d3bde19
+oid sha256:5c30d56d1e5b3de7f5358c607038a976c0e55b139526af0ae83b39ad89800afe
 size 6520