Training in progress, step 4000

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e317ad223be2f7bb6f6d75fe7fdba37e02d4e3f6ff669cc7b1478197da5b1be
+oid sha256:b112ce39756b7c05081ae163cb9098363f386978280821af68a6fa5286669b47
 size 500897101

checkpoint-3000/adapter_model/README.md

@@ -0,0 +1,20 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-3000/adapter_model/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-3000/adapter_model/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e317ad223be2f7bb6f6d75fe7fdba37e02d4e3f6ff669cc7b1478197da5b1be
+size 500897101

checkpoint-4000/README.md

@@ -0,0 +1,20 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-4000/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-4000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b112ce39756b7c05081ae163cb9098363f386978280821af68a6fa5286669b47
+size 500897101

checkpoint-4000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fdc55fafb7bf47aa4e1a0f25d468aa75b7bc6b534c14b14961026310ac3f0ad
+size 1001723453

checkpoint-4000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:867f0e937027681f139b8a498470607164d5c933a2c3fa2808d34789d9c719e3
+size 14575

checkpoint-4000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fd70f56badc94591a6f4795d3a86a8ce5d7d40606c861b20219208c04773382
+size 627

checkpoint-4000/trainer_state.json

@@ -0,0 +1,288 @@
+{
+  "best_metric": 0.726176917552948,
+  "best_model_checkpoint": "./qlora-out/checkpoint-4000",
+  "epoch": 0.14913687036277543,
+  "global_step": 4000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019999938245325715,
+      "loss": 0.9023,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00019999724773356797,
+      "loss": 0.8027,
+      "step": 200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0001999935882494411,
+      "loss": 0.8041,
+      "step": 300
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00019998840405667672,
+      "loss": 0.7944,
+      "step": 400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00019998169523432365,
+      "loss": 0.81,
+      "step": 500
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0001999734618846785,
+      "loss": 0.7855,
+      "step": 600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00019996370413328385,
+      "loss": 0.7849,
+      "step": 700
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00019995242212892653,
+      "loss": 0.7564,
+      "step": 800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00019993961604363532,
+      "loss": 0.7724,
+      "step": 900
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00019992528607267815,
+      "loss": 0.7308,
+      "step": 1000
+    },
+    {
+      "epoch": 0.04,
+      "eval_loss": 0.7677998542785645,
+      "eval_runtime": 1774.3517,
+      "eval_samples_per_second": 0.305,
+      "eval_steps_per_second": 0.305,
+      "step": 1000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0001999094324345594,
+      "loss": 0.7844,
+      "step": 1100
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00019989205537101633,
+      "loss": 0.7668,
+      "step": 1200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00019987315514701553,
+      "loss": 0.7727,
+      "step": 1300
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00019985273205074878,
+      "loss": 0.7467,
+      "step": 1400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00019983078639362883,
+      "loss": 0.7516,
+      "step": 1500
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00019980731851028445,
+      "loss": 0.7267,
+      "step": 1600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0001997823287585554,
+      "loss": 0.7632,
+      "step": 1700
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000199755817519487,
+      "loss": 0.7392,
+      "step": 1800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00019972778519732436,
+      "loss": 0.7528,
+      "step": 1900
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0001996982322195061,
+      "loss": 0.725,
+      "step": 2000
+    },
+    {
+      "epoch": 0.07,
+      "eval_loss": 0.7452704310417175,
+      "eval_runtime": 1787.7554,
+      "eval_samples_per_second": 0.303,
+      "eval_steps_per_second": 0.303,
+      "step": 2000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00019966715903665795,
+      "loss": 0.7234,
+      "step": 2100
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00019963456612258576,
+      "loss": 0.754,
+      "step": 2200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00019960045397426841,
+      "loss": 0.7856,
+      "step": 2300
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00019956482311185006,
+      "loss": 0.7387,
+      "step": 2400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00019952767407863245,
+      "loss": 0.7309,
+      "step": 2500
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00019948900744106633,
+      "loss": 0.7232,
+      "step": 2600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00019944882378874316,
+      "loss": 0.7406,
+      "step": 2700
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0001994071237343858,
+      "loss": 0.7166,
+      "step": 2800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00019936390791383936,
+      "loss": 0.7308,
+      "step": 2900
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00019931917698606143,
+      "loss": 0.7288,
+      "step": 3000
+    },
+    {
+      "epoch": 0.11,
+      "eval_loss": 0.7343490123748779,
+      "eval_runtime": 1770.9966,
+      "eval_samples_per_second": 0.306,
+      "eval_steps_per_second": 0.306,
+      "step": 3000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00019927293163311206,
+      "loss": 0.7236,
+      "step": 3100
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00019922517256014337,
+      "loss": 0.716,
+      "step": 3200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00019917590049538874,
+      "loss": 0.7564,
+      "step": 3300
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00019912511619015177,
+      "loss": 0.7082,
+      "step": 3400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00019907282041879484,
+      "loss": 0.7103,
+      "step": 3500
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00019901901397872715,
+      "loss": 0.7457,
+      "step": 3600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0001989636976903928,
+      "loss": 0.7076,
+      "step": 3700
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0001989068723972581,
+      "loss": 0.7217,
+      "step": 3800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00019884853896579873,
+      "loss": 0.7175,
+      "step": 3900
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0001987886982854866,
+      "loss": 0.7083,
+      "step": 4000
+    },
+    {
+      "epoch": 0.15,
+      "eval_loss": 0.726176917552948,
+      "eval_runtime": 1765.3933,
+      "eval_samples_per_second": 0.307,
+      "eval_steps_per_second": 0.307,
+      "step": 4000
+    }
+  ],
+  "max_steps": 80463,
+  "num_train_epochs": 3,
+  "total_flos": 1.1223871732835942e+18,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-4000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a281372a257be5e7ebbea8ac16ec51e707b224a5ed57c48e2f69549c5a031d8
+size 4027