Model save

README.md

@@ -1,11 +1,8 @@
 ---
-license: mit
-base_model: HuggingFaceH4/mistral-7b-sft-beta
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
 - generated_from_trainer
-datasets:
-- Anthropic/hh-rlhf
 model-index:
 - name: zephyr-7b-dpo-lora
   results: []
@@ -16,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-lora
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the Anthropic/hh-rlhf dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7922
-- Rewards/chosen: -6.3032
-- Rewards/rejected: -8.1308
-- Rewards/accuracies: 0.7062
-- Rewards/margins: 1.8276
-- Logps/rejected: -238.5443
-- Logps/chosen: -222.5519
-- Logits/rejected: -2.6859
-- Logits/chosen: -2.6948
+- Loss: 0.6488
+- Rewards/chosen: 0.0341
+- Rewards/rejected: -0.0820
+- Rewards/accuracies: 0.7109
+- Rewards/margins: 0.1161
+- Logps/rejected: -224.8079
+- Logps/chosen: -271.6428
+- Logits/rejected: -3.0562
+- Logits/chosen: -3.0761
 
 ## Model description
 
@@ -45,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0002
+- learning_rate: 5e-07
 - train_batch_size: 16
 - eval_batch_size: 16
 - seed: 42
@@ -56,20 +53,18 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 3
+- num_epochs: 1.0
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5668        | 1.0   | 1246 | 0.5712          | -0.9288        | -1.6660          | 0.6922             | 0.7372          | -173.8961      | -168.8080    | -2.7862         | -2.7998       |
-| 0.2986        | 2.0   | 2492 | 0.6037          | -2.4129        | -3.4852          | 0.7201             | 1.0722          | -192.0881      | -183.6498    | -2.8543         | -2.8663       |
-| 0.0973        | 3.0   | 3738 | 0.7922          | -6.3032        | -8.1308          | 0.7062             | 1.8276          | -238.5443      | -222.5519    | -2.6859         | -2.6948       |
+| 0.6488        | 1.0   | 485  | 0.6488          | 0.0341         | -0.0820          | 0.7109             | 0.1161          | -224.8079      | -271.6428    | -3.0562         | -3.0761       |
 
 
 ### Framework versions
 
 - Transformers 4.35.0
-- Pytorch 2.1.1+cu121
+- Pytorch 2.1.0+cu121
 - Datasets 2.14.6
 - Tokenizers 0.14.1

adapter_config.json

@@ -17,9 +17,9 @@
   "revision": null,
   "target_modules": [
     "o_proj",
-    "v_proj",
     "q_proj",
-    "k_proj"
+    "k_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30289b286308940696068f0ad4375260c6bfb714350a71c97ef8a9e9582042fb
+oid sha256:df593ca3ae7bf680c032ec8295da32728b125748963b4665b4da1dfd2c2164db
 size 109086672

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 3.0,
-    "eval_logits/chosen": -2.694794178009033,
-    "eval_logits/rejected": -2.6858575344085693,
-    "eval_logps/chosen": -222.5519256591797,
-    "eval_logps/rejected": -238.54434204101562,
-    "eval_loss": 0.7921916842460632,
-    "eval_rewards/accuracies": 0.7061567306518555,
-    "eval_rewards/chosen": -6.303151607513428,
-    "eval_rewards/margins": 1.8276441097259521,
-    "eval_rewards/rejected": -8.1307954788208,
-    "eval_runtime": 852.8388,
-    "eval_samples": 8483,
-    "eval_samples_per_second": 9.947,
-    "eval_steps_per_second": 0.079,
-    "train_loss": 0.3484600728976656,
-    "train_runtime": 105269.9843,
-    "train_samples": 159397,
-    "train_samples_per_second": 4.543,
-    "train_steps_per_second": 0.036
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.07612943649292,
+    "eval_logits/rejected": -3.056239604949951,
+    "eval_logps/chosen": -271.6427917480469,
+    "eval_logps/rejected": -224.8079376220703,
+    "eval_loss": 0.6488261818885803,
+    "eval_rewards/accuracies": 0.7109375,
+    "eval_rewards/chosen": 0.034067459404468536,
+    "eval_rewards/margins": 0.11610361933708191,
+    "eval_rewards/rejected": -0.08203616738319397,
+    "eval_runtime": 254.1478,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 7.869,
+    "eval_steps_per_second": 0.063,
+    "train_loss": 0.6667533972828659,
+    "train_runtime": 15505.6746,
+    "train_samples": 61966,
+    "train_samples_per_second": 3.996,
+    "train_steps_per_second": 0.031
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 3.0,
-    "eval_logits/chosen": -2.694794178009033,
-    "eval_logits/rejected": -2.6858575344085693,
-    "eval_logps/chosen": -222.5519256591797,
-    "eval_logps/rejected": -238.54434204101562,
-    "eval_loss": 0.7921916842460632,
-    "eval_rewards/accuracies": 0.7061567306518555,
-    "eval_rewards/chosen": -6.303151607513428,
-    "eval_rewards/margins": 1.8276441097259521,
-    "eval_rewards/rejected": -8.1307954788208,
-    "eval_runtime": 852.8388,
-    "eval_samples": 8483,
-    "eval_samples_per_second": 9.947,
-    "eval_steps_per_second": 0.079
+    "epoch": 1.0,
+    "eval_logits/chosen": -3.07612943649292,
+    "eval_logits/rejected": -3.056239604949951,
+    "eval_logps/chosen": -271.6427917480469,
+    "eval_logps/rejected": -224.8079376220703,
+    "eval_loss": 0.6488261818885803,
+    "eval_rewards/accuracies": 0.7109375,
+    "eval_rewards/chosen": 0.034067459404468536,
+    "eval_rewards/margins": 0.11610361933708191,
+    "eval_rewards/rejected": -0.08203616738319397,
+    "eval_runtime": 254.1478,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 7.869,
+    "eval_steps_per_second": 0.063
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "train_loss": 0.3484600728976656,
-    "train_runtime": 105269.9843,
-    "train_samples": 159397,
-    "train_samples_per_second": 4.543,
-    "train_steps_per_second": 0.036
+    "epoch": 1.0,
+    "train_loss": 0.6667533972828659,
+    "train_runtime": 15505.6746,
+    "train_samples": 61966,
+    "train_samples_per_second": 3.996,
+    "train_steps_per_second": 0.031
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea442d361aad58962e30eb6b5e88eb41a119a6bd1f02593c97d61b2fe3be8197
+oid sha256:85762387de695f46895984cce9b354002e7bf3e93e34a5f99eed66dcb69e0b22
 size 5688