Training in progress, step 3200

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:604d08ed43fee049ea3799c6450a4a1d5f8cc6e58ddf33f377215e77f03ec769
+oid sha256:db5a92be0cd5f8b38b328e0f82e62452a3fa7b5052a0a1f93fd8c4b1dd18b7a7
 size 500897101

{checkpoint-2800 → checkpoint-3100/adapter_model}/adapter_config.json

@@ -14,13 +14,13 @@
   "r": 32,
   "revision": null,
   "target_modules": [
+    "down_proj",
+    "k_proj",
+    "gate_proj",
     "v_proj",
     "o_proj",
-    "down_proj",
     "q_proj",
-    "gate_proj",
-    "up_proj",
-    "k_proj"
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

{checkpoint-2800 → checkpoint-3100/adapter_model}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ea36cf8e55409a14da3fc648b1dfbd6758083b72318e4742d7d9efc2963c331
+oid sha256:604d08ed43fee049ea3799c6450a4a1d5f8cc6e58ddf33f377215e77f03ec769
 size 500897101

checkpoint-3200/README.md

@@ -0,0 +1,20 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-3200/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-3200/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db5a92be0cd5f8b38b328e0f82e62452a3fa7b5052a0a1f93fd8c4b1dd18b7a7
+size 500897101

{checkpoint-2800 → checkpoint-3200}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ad547cbbf1779bc43d5f541ea80ff23dfadca360f5b7fbe6b9210bdabd5b0a3
+oid sha256:734792a711547c325f20f564fba3fc1dafb5a5e2eb00a8e88a628060b50ad4e0
 size 1001752701

{checkpoint-2800 → checkpoint-3200}/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54ceaf70d5ffe3e642b177d8e34d1f05742963af64a1bcc361535a9360b397c4
+oid sha256:47493f691983492eb548e89c4bc00a908f9336311a3cdb9b6d4d0436862a7afe
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9607a8c2ef03708a5690aea65c1585d4ddbd8e01d7e279a5781ba30e8f1917ff
+oid sha256:7fd2d4258cdbe26a4276e6adf71ed40f5fb449eeb7efadbcb60ee6c3eb243b4a
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_10.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8681f4b5196d1fde8691794cb783ea1dab72c5077dbcf5394c6998bd4136e731
+oid sha256:a1a66a04d36cd4ced3163ee7de7c3f532444837ca748b2c7402468c7cf9ac0cc
 size 27789

{checkpoint-2800 → checkpoint-3200}/rng_state_11.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd6cb0b4fef9e193eda245bff5fbcc8df9769fc61966e5f1cdf256150779c1da
+oid sha256:6fd816f782b7158df45bbb3a0847b7e567cbc8246b73c484a51cea72f837a615
 size 27789

{checkpoint-2800 → checkpoint-3200}/rng_state_12.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b66ef66b2b0e1dce0083092ba63e49bdf210999432342b9c2a3993ca381ca353
+oid sha256:a5c0e7d21944297ce2889bf41e83eaf69e29b09934c6e5396b81f21c8d554008
 size 27789

{checkpoint-2800 → checkpoint-3200}/rng_state_13.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5cec3ba0fdf94eca1cdf6294a4f5fddb5e3c4f431222a020f6b0acac68bcd0b2
+oid sha256:a4a856ae9cc1a21a8770367e26ba40b6eb42e788a7f1046e05e030a2571733a3
 size 27789

{checkpoint-2800 → checkpoint-3200}/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c53913af6dbe6026773029476e95c35bab12c198a7f90d679010f0fd21198df
+oid sha256:21fe838aa7244c39844b0cc17fda41f0bf30a92e8c365cbf60e9909dc9539b7a
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:811ad146e12ad4449d58b59f6fcb92f298ffd443e1a43261d70a55e3eb6343d9
+oid sha256:c73d64b42d450cb0cd9d780bb5d76b4532c040febbd3e57c1a7e507f307c6919
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5da3f408ca3cd0e89a66524e7f4846e322256ba24237784526bdc51b9ede18b
+oid sha256:dd75d438f8c213a76df48ea475d4e8d725ef60a192611b3a3b5156b54d9bd3aa
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c659f82e6a038ab94307ae424a29fedf3767fe4eabecd42c8ea9b1fb19e36fc
+oid sha256:a92b4282b70600e8661d984a46aae33c95db7e5a45a98aa75d1ab7fe809f0e87
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d5c97078c0cd3e6af8628033620fd2ade2aea530684443c37b10455b6826ae1
+oid sha256:60a6277ffeade78c02a3c80e695be2609d9c03891fa74c239da9236a5d8a411f
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dd4ff6e318b6e48ea5386ceaf02104c9a56c201311646c4a4e1f4c3be379fe7
+oid sha256:04ceb85b25ddf1ff875502c31946bedfdea45ff9b753957a01149a9fc85fe4aa
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_8.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31d89de68577a860ceffdb664eb8eb7a44e5a8dd6aa62c351c0caf7b931e6a02
+oid sha256:7647a4e90848b95f8b12db8456e13797e34b5cb91f94f229dfe860ef88396c11
 size 27772

{checkpoint-2800 → checkpoint-3200}/rng_state_9.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76b3e7281559d95415194847aba99c3c754c2f0ad8f10ac4eb274f044a763cf1
+oid sha256:f4297218a6c8fddf475d7093a33dde8e1024bb348b2eacb353ad76d4cb02c652
 size 27772

{checkpoint-2800 → checkpoint-3200}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0008ae7bce85f92437f0e149969fba2477d40c35a55a3a590f361670b6eb7290
+oid sha256:c684e0d6e24a062502c981434add298c33b176147f87f977608afcf00cc005c3
 size 627

{checkpoint-2800 → checkpoint-3200}/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0856921287320667,
-  "global_step": 2800,
+  "epoch": 1.240791004265219,
+  "global_step": 3200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -670,11 +670,115 @@
       "learning_rate": 0.0001422730849785107,
       "loss": 0.8091,
       "step": 2800
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 0.0001413497689423539,
+      "loss": 0.8067,
+      "step": 2825
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 0.00014042218094512755,
+      "loss": 0.8046,
+      "step": 2850
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 0.00013949041681855985,
+      "loss": 0.8053,
+      "step": 2875
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 0.0001385545728258264,
+      "loss": 0.8075,
+      "step": 2900
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 0.0001376147456516055,
+      "loss": 0.8015,
+      "step": 2925
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 0.00013667103239208903,
+      "loss": 0.8016,
+      "step": 2950
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 0.00013572353054495126,
+      "loss": 0.8029,
+      "step": 2975
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 0.0001347723379992762,
+      "loss": 0.8017,
+      "step": 3000
+    },
+    {
+      "epoch": 1.16,
+      "eval_loss": 0.8229297995567322,
+      "eval_runtime": 59.3398,
+      "eval_samples_per_second": 12.302,
+      "eval_steps_per_second": 0.893,
+      "step": 3000
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 0.0001338175530254443,
+      "loss": 0.8049,
+      "step": 3025
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 0.00013285927426497985,
+      "loss": 0.8027,
+      "step": 3050
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 0.00013189760072036008,
+      "loss": 0.8028,
+      "step": 3075
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 0.0001309326317447869,
+      "loss": 0.8021,
+      "step": 3100
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 0.00012996446703192257,
+      "loss": 0.8033,
+      "step": 3125
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 0.00012899320660558986,
+      "loss": 0.8016,
+      "step": 3150
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 0.00012801895080943846,
+      "loss": 0.7995,
+      "step": 3175
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 0.0001270418002965782,
+      "loss": 0.799,
+      "step": 3200
     }
   ],
   "max_steps": 7737,
   "num_train_epochs": 3,
-  "total_flos": 1.205858669606155e+19,
+  "total_flos": 1.3780535807691457e+19,
   "trial_name": null,
   "trial_params": null
 }

{checkpoint-2800 → checkpoint-3200}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1240323c6a8f49a8a8b1664a44594fc01574f066beb8bd9d04ff53bb19046038
+oid sha256:7292138fecd854f5f17371c439bbd450ee3c48e738b75818b778a55f4e26ef57
 size 4027