Model save

README.md

@@ -1,4 +1,6 @@
 ---
+license: gemma
+base_model: tanliboy/zephyr-7b-gemma-sft
 tags:
 - trl
 - dpo
@@ -13,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-gemma-dpo
 
-This model was trained from scratch on an unknown dataset.
+This model is a fine-tuned version of [tanliboy/zephyr-7b-gemma-sft](https://huggingface.co/tanliboy/zephyr-7b-gemma-sft) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4812
-- Rewards/chosen: -0.1423
-- Rewards/rejected: -1.2147
-- Rewards/accuracies: 0.7083
-- Rewards/margins: 1.0724
-- Logps/rejected: -719.2219
-- Logps/chosen: -699.1321
-- Logits/rejected: 152.4098
-- Logits/chosen: 152.7205
+- Loss: 0.4722
+- Rewards/chosen: -0.0658
+- Rewards/rejected: -1.2673
+- Rewards/accuracies: 0.7396
+- Rewards/margins: 1.2015
+- Logps/rejected: -720.2745
+- Logps/chosen: -697.6023
+- Logits/rejected: 152.9660
+- Logits/chosen: 153.1356
 
 ## Model description
 
@@ -60,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1441        | 1.8957 | 100  | 0.4812          | -0.1423        | -1.2147          | 0.7083             | 1.0724          | -719.2219      | -699.1321    | 152.4098        | 152.7205      |
+| 0.1424        | 1.8957 | 100  | 0.4722          | -0.0658        | -1.2673          | 0.7396             | 1.2015          | -720.2745      | -697.6023    | 152.9660        | 153.1356      |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.38755885110451627,
-    "train_runtime": 2329.0461,
+    "train_loss": 0.3875045489806395,
+    "train_runtime": 2331.3598,
     "train_samples": 6750,
-    "train_samples_per_second": 5.796,
+    "train_samples_per_second": 5.791,
     "train_steps_per_second": 0.045
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/home/litan/alignment-handbook/models/zephyr-7b-gemma-sft",
+  "_name_or_path": "tanliboy/zephyr-7b-gemma-sft",
   "architectures": [
     "GemmaForCausalLM"
   ],

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e294f891ceb65882f57743942bc3dc24c4b2b502bbb076096ec25e92661e741
+oid sha256:563e4e67cd479b49935cacef2e942c6012884ff407662bf19bc5df331e539025
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4204106f6e2d2ff00e095e8568ce50ddc74989dbc5991ea07cb8629ad54e031b
+oid sha256:60905c763b2b5422bd986dfd9cd9ca91c5cf5cf91f20cec4c90a53de04de867f
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c318a4f435751720bc413d8ee12e30b1bd2e08cc85b889991c2328190361a7da
+oid sha256:c9b965885821657872f7222d640a45d5812b610e3df4fd23dc6d0b95f41e688c
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c840fbdf9fdb4f8b1d76fcba6b20d0bd75ad4f7bbc5eac9198663dbf2a6c958c
+oid sha256:bb7604d8de752d9cab47da8539302cac4f95c5754577653ecc748f57930bea65
 size 2113988336

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.38755885110451627,
-    "train_runtime": 2329.0461,
+    "train_loss": 0.3875045489806395,
+    "train_runtime": 2331.3598,
     "train_samples": 6750,
-    "train_samples_per_second": 5.796,
+    "train_samples_per_second": 5.791,
     "train_steps_per_second": 0.045
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 99.03845755243859,
+      "grad_norm": 99.03995946284127,
       "learning_rate": 4.545454545454545e-08,
       "logits/chosen": 163.72256469726562,
       "logits/rejected": 157.14466857910156,
@@ -25,177 +25,177 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 109.21636585235684,
+      "grad_norm": 111.11388003021844,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 171.64620971679688,
-      "logits/rejected": 172.84539794921875,
-      "logps/chosen": -742.3294067382812,
-      "logps/rejected": -781.19384765625,
-      "loss": 0.7155,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": 0.017745885998010635,
-      "rewards/margins": 0.0211933646351099,
-      "rewards/rejected": -0.0034474804997444153,
+      "logits/chosen": 171.66250610351562,
+      "logits/rejected": 172.8583221435547,
+      "logps/chosen": -742.216064453125,
+      "logps/rejected": -781.2522583007812,
+      "loss": 0.7117,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.023412303999066353,
+      "rewards/margins": 0.02978678233921528,
+      "rewards/rejected": -0.006374475546181202,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 81.70841250586862,
+      "grad_norm": 82.52463810141795,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": 162.9903106689453,
-      "logits/rejected": 165.7174835205078,
-      "logps/chosen": -710.62255859375,
-      "logps/rejected": -750.2293701171875,
-      "loss": 0.6502,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.47601765394210815,
-      "rewards/margins": 0.17595532536506653,
-      "rewards/rejected": 0.300062358379364,
+      "logits/chosen": 163.12501525878906,
+      "logits/rejected": 165.84164428710938,
+      "logps/chosen": -709.082275390625,
+      "logps/rejected": -749.2286376953125,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.553031325340271,
+      "rewards/margins": 0.20293028652668,
+      "rewards/rejected": 0.35010096430778503,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 70.21647250732705,
+      "grad_norm": 69.67853625499806,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": 183.96597290039062,
-      "logits/rejected": 176.80657958984375,
-      "logps/chosen": -712.5685424804688,
-      "logps/rejected": -738.5382080078125,
-      "loss": 0.582,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.9081576466560364,
-      "rewards/margins": 0.5097990036010742,
-      "rewards/rejected": 0.3983585834503174,
+      "logits/chosen": 184.1649932861328,
+      "logits/rejected": 176.99354553222656,
+      "logps/chosen": -710.6904907226562,
+      "logps/rejected": -736.7250366210938,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 1.0020567178726196,
+      "rewards/margins": 0.513039767742157,
+      "rewards/rejected": 0.4890168309211731,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 70.35328347282919,
+      "grad_norm": 72.75906612350751,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": 167.0669708251953,
-      "logits/rejected": 155.26910400390625,
-      "logps/chosen": -696.257080078125,
-      "logps/rejected": -698.9947509765625,
-      "loss": 0.571,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.4749051034450531,
-      "rewards/margins": 0.8314126133918762,
-      "rewards/rejected": -0.35650748014450073,
+      "logits/chosen": 167.30398559570312,
+      "logits/rejected": 155.48980712890625,
+      "logps/chosen": -697.3994140625,
+      "logps/rejected": -700.686767578125,
+      "loss": 0.5672,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.41778701543807983,
+      "rewards/margins": 0.858893871307373,
+      "rewards/rejected": -0.44110679626464844,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 89.79206485749032,
+      "grad_norm": 91.77276628667218,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": 181.7749481201172,
-      "logits/rejected": 182.71652221679688,
-      "logps/chosen": -772.6339721679688,
-      "logps/rejected": -824.01171875,
-      "loss": 0.4953,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.038030486553907394,
-      "rewards/margins": 0.9203092455863953,
-      "rewards/rejected": -0.8822787404060364,
+      "logits/chosen": 182.02633666992188,
+      "logits/rejected": 182.9945068359375,
+      "logps/chosen": -771.2996826171875,
+      "logps/rejected": -821.66943359375,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.10474413633346558,
+      "rewards/margins": 0.8699092864990234,
+      "rewards/rejected": -0.7651651501655579,
       "step": 50
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 29.72925097875123,
+      "grad_norm": 29.047916941624063,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": 174.12176513671875,
-      "logits/rejected": 179.2202606201172,
-      "logps/chosen": -724.5867309570312,
-      "logps/rejected": -795.8361206054688,
-      "loss": 0.2765,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.7417961955070496,
-      "rewards/margins": 2.4216690063476562,
-      "rewards/rejected": -1.6798728704452515,
+      "logits/chosen": 174.08924865722656,
+      "logits/rejected": 179.21060180664062,
+      "logps/chosen": -725.1399536132812,
+      "logps/rejected": -794.7996215820312,
+      "loss": 0.2763,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.714134693145752,
+      "rewards/margins": 2.342179298400879,
+      "rewards/rejected": -1.6280447244644165,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 29.451582781492476,
+      "grad_norm": 28.169902323181958,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": 169.59214782714844,
-      "logits/rejected": 167.35098266601562,
-      "logps/chosen": -683.1103515625,
-      "logps/rejected": -780.7704467773438,
-      "loss": 0.177,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.8972679972648621,
-      "rewards/margins": 2.7340922355651855,
-      "rewards/rejected": -1.836824655532837,
+      "logits/chosen": 169.7666778564453,
+      "logits/rejected": 167.54342651367188,
+      "logps/chosen": -683.6437377929688,
+      "logps/rejected": -783.1070556640625,
+      "loss": 0.1734,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.8705979585647583,
+      "rewards/margins": 2.8242533206939697,
+      "rewards/rejected": -1.9536556005477905,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 21.98036303376898,
+      "grad_norm": 24.1551722040215,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": 159.87899780273438,
-      "logits/rejected": 172.0943603515625,
-      "logps/chosen": -684.193359375,
-      "logps/rejected": -807.0607299804688,
-      "loss": 0.1583,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.6033011674880981,
-      "rewards/margins": 3.1811580657958984,
-      "rewards/rejected": -2.5778567790985107,
+      "logits/chosen": 160.0974578857422,
+      "logits/rejected": 172.2356414794922,
+      "logps/chosen": -684.6734619140625,
+      "logps/rejected": -806.4854125976562,
+      "loss": 0.1605,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.5792978405952454,
+      "rewards/margins": 3.1283910274505615,
+      "rewards/rejected": -2.549093246459961,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 30.02402881967663,
+      "grad_norm": 25.406031573666652,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": 158.72433471679688,
-      "logits/rejected": 170.1360626220703,
-      "logps/chosen": -715.9267578125,
-      "logps/rejected": -813.0823974609375,
-      "loss": 0.1439,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 0.6878162622451782,
-      "rewards/margins": 3.3359310626983643,
-      "rewards/rejected": -2.6481146812438965,
+      "logits/chosen": 158.98464965820312,
+      "logits/rejected": 170.3443145751953,
+      "logps/chosen": -717.3196411132812,
+      "logps/rejected": -813.6456298828125,
+      "loss": 0.1493,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.6181727647781372,
+      "rewards/margins": 3.2944443225860596,
+      "rewards/rejected": -2.6762712001800537,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 24.732786108992524,
+      "grad_norm": 22.259191659621163,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": 162.75802612304688,
-      "logits/rejected": 165.20050048828125,
-      "logps/chosen": -721.2442016601562,
-      "logps/rejected": -793.7107543945312,
-      "loss": 0.1441,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 0.6896085143089294,
-      "rewards/margins": 2.8235270977020264,
-      "rewards/rejected": -2.1339187622070312,
+      "logits/chosen": 163.0000762939453,
+      "logits/rejected": 165.51370239257812,
+      "logps/chosen": -720.561767578125,
+      "logps/rejected": -794.7210693359375,
+      "loss": 0.1424,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 0.7237287759780884,
+      "rewards/margins": 2.908165454864502,
+      "rewards/rejected": -2.184436559677124,
       "step": 100
     },
     {
       "epoch": 1.8957345971563981,
-      "eval_logits/chosen": 152.7205047607422,
-      "eval_logits/rejected": 152.40980529785156,
-      "eval_logps/chosen": -699.132080078125,
-      "eval_logps/rejected": -719.221923828125,
-      "eval_loss": 0.48118945956230164,
-      "eval_rewards/accuracies": 0.7083333134651184,
-      "eval_rewards/chosen": -0.1422799676656723,
-      "eval_rewards/margins": 1.0724023580551147,
-      "eval_rewards/rejected": -1.2146824598312378,
-      "eval_runtime": 105.6518,
-      "eval_samples_per_second": 7.099,
-      "eval_steps_per_second": 0.227,
+      "eval_logits/chosen": 153.1355743408203,
+      "eval_logits/rejected": 152.9660186767578,
+      "eval_logps/chosen": -697.602294921875,
+      "eval_logps/rejected": -720.2744750976562,
+      "eval_loss": 0.47219032049179077,
+      "eval_rewards/accuracies": 0.7395833134651184,
+      "eval_rewards/chosen": -0.06579157710075378,
+      "eval_rewards/margins": 1.2015198469161987,
+      "eval_rewards/rejected": -1.267311453819275,
+      "eval_runtime": 116.4984,
+      "eval_samples_per_second": 6.438,
+      "eval_steps_per_second": 0.206,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.38755885110451627,
-      "train_runtime": 2329.0461,
-      "train_samples_per_second": 5.796,
+      "train_loss": 0.3875045489806395,
+      "train_runtime": 2331.3598,
+      "train_samples_per_second": 5.791,
       "train_steps_per_second": 0.045
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c24086242ff952a27d9b8b7a937bb99318bdbf5629af339f0a6ac342e25fb1ad
+oid sha256:62f57065711105b6cf3f7022479eff20f007bd391bba46b9f0c9d6d7fcd8a2ae
 size 6264