Model save

README.md

@@ -2,15 +2,9 @@
 license: other
 base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- argilla/dpo-mix-7k
 model-index:
 - name: zephyr-7b-gemma-dpo
   results: []
@@ -19,20 +13,9 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/sanqiang/wdpo/runs/cbwapxr3)
 # zephyr-7b-gemma-dpo
 
-This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the argilla/dpo-mix-7k dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.4694
-- Rewards/chosen: -3.9430
-- Rewards/rejected: -5.5568
-- Rewards/accuracies: 0.7292
-- Rewards/margins: 1.6137
-- Logps/rejected: -473.0876
-- Logps/chosen: -441.5945
-- Logits/rejected: 88.3937
-- Logits/chosen: 94.2780
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the None dataset.
 
 ## Model description
 
@@ -52,14 +35,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 2
-- eval_batch_size: 4
+- train_batch_size: 1
+- eval_batch_size: 1
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 8
+- gradient_accumulation_steps: 16
 - total_train_batch_size: 128
-- total_eval_batch_size: 32
+- total_eval_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -67,14 +50,11 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1534        | 1.8957 | 100  | 0.4631          | -3.9815        | -5.5353          | 0.7083             | 1.5538          | -472.6590      | -442.3637    | 88.4677         | 94.3385       |
 
 
 ### Framework versions
 
-- Transformers 4.41.0
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu121
 - Datasets 2.14.6
 - Tokenizers 0.19.1

all_results.json

@@ -1,22 +1,9 @@
 {
     "epoch": 1.971563981042654,
-    "eval_logits/chosen": 94.27800750732422,
-    "eval_logits/rejected": 88.3936767578125,
-    "eval_logps/chosen": -441.594482421875,
-    "eval_logps/rejected": -473.087646484375,
-    "eval_loss": 0.4694259464740753,
-    "eval_rewards/accuracies": 0.7291666865348816,
-    "eval_rewards/chosen": -3.94303035736084,
-    "eval_rewards/margins": 1.613731026649475,
-    "eval_rewards/rejected": -5.556761264801025,
-    "eval_runtime": 20.9081,
-    "eval_samples": 750,
-    "eval_samples_per_second": 35.871,
-    "eval_steps_per_second": 1.148,
     "total_flos": 0.0,
-    "train_loss": 0.38818228932527393,
-    "train_runtime": 1555.646,
+    "train_loss": 0.39618923515081406,
+    "train_runtime": 6217.6844,
     "train_samples": 6750,
-    "train_samples_per_second": 8.678,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 2.171,
+    "train_steps_per_second": 0.017
 }

config.json

@@ -23,7 +23,7 @@
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.0",
-  "use_cache": true,
+  "transformers_version": "4.41.1",
+  "use_cache": false,
   "vocab_size": 256000
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 2,
   "eos_token_id": 1,
   "pad_token_id": 0,
-  "transformers_version": "4.41.0"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:907718dc7abf77c674dfe5561577edaa20f6fa1f02bf06b9de7a03e0c15e2485
+oid sha256:8173c05d1efe08c4065544d793175322eb2200be1887726dea6644d5277d73ac
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4670809bdabbdfeeaa959616f541ed35c881f72174d92315b90b89b7c7ed884a
+oid sha256:02256ff0bcb27472feccfb9450965b57ad05a4ab4aa4bec3b416265f8261aaf8
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9782ec18e737bd2641fe5faa1dfc877b1e190168f373747e733b300ba25c56e
+oid sha256:f4eeebdfd0c17424180ff75420235e53fc228ffae6dac9af48c4fd07b803d870
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15dd432be6f7df2743b1bef22516b2f769aa7e7804086c1875c9abd5f2f2e0b5
+oid sha256:f8ea7640a19dbd05fd2186e383d4a64ec43af0f5b7299d629a37ff306fc7e9f8
 size 2113988336

runs/May24_04-18-29_pytorchjob-summarization-debugkd-ifxb-sanqiang-zhao-master-0/events.out.tfevents.1716524387.pytorchjob-summarization-debugkd-ifxb-sanqiang-zhao-master-0.8891.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac4ecd006f7c6c3cd9fc1ab0bdf028b9f97a21b144976016c57eade392f96f90
+size 12870

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.38818228932527393,
-    "train_runtime": 1555.646,
+    "train_loss": 0.39618923515081406,
+    "train_runtime": 6217.6844,
     "train_samples": 6750,
-    "train_samples_per_second": 8.678,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 2.171,
+    "train_steps_per_second": 0.017
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 1.971563981042654,
-  "eval_steps": 100,
+  "eval_steps": 500,
   "global_step": 104,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 134.93724401851352,
+      "grad_norm": 135.46242416026297,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": 117.67350769042969,
-      "logits/rejected": 126.90988159179688,
-      "logps/chosen": -336.5020751953125,
-      "logps/rejected": -438.0943298339844,
+      "logits/chosen": 203.78909301757812,
+      "logits/rejected": 182.07696533203125,
+      "logps/chosen": -434.955322265625,
+      "logps/rejected": -429.50384521484375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,178 +25,162 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 136.34386152980758,
+      "grad_norm": 153.21007203142685,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 134.7221221923828,
-      "logits/rejected": 138.1984100341797,
-      "logps/chosen": -395.924560546875,
-      "logps/rejected": -439.0712890625,
-      "loss": 0.7055,
-      "rewards/accuracies": 0.4097222089767456,
-      "rewards/chosen": -0.003234411356970668,
-      "rewards/margins": -0.03947298228740692,
-      "rewards/rejected": 0.036238569766283035,
+      "logits/chosen": 172.51792907714844,
+      "logits/rejected": 181.91905212402344,
+      "logps/chosen": -379.5159912109375,
+      "logps/rejected": -448.1414794921875,
+      "loss": 0.7124,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.027371780946850777,
+      "rewards/margins": 0.008838895708322525,
+      "rewards/rejected": 0.01853288896381855,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 135.65125923828373,
+      "grad_norm": 142.08069477633126,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": 121.62422180175781,
-      "logits/rejected": 125.15272521972656,
-      "logps/chosen": -369.32891845703125,
-      "logps/rejected": -422.75689697265625,
-      "loss": 0.6228,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.170469269156456,
-      "rewards/margins": 0.30520111322402954,
-      "rewards/rejected": -0.13473184406757355,
+      "logits/chosen": 173.6682891845703,
+      "logits/rejected": 175.75106811523438,
+      "logps/chosen": -381.7967834472656,
+      "logps/rejected": -430.2057189941406,
+      "loss": 0.6304,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.19567528367042542,
+      "rewards/margins": 0.36299628019332886,
+      "rewards/rejected": -0.16732101142406464,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 114.07857539812969,
+      "grad_norm": 120.91669313906786,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": 141.53656005859375,
-      "logits/rejected": 135.02110290527344,
-      "logps/chosen": -418.7715759277344,
-      "logps/rejected": -464.8955078125,
-      "loss": 0.56,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.2944904565811157,
-      "rewards/margins": 0.8948480486869812,
-      "rewards/rejected": -2.189338445663452,
+      "logits/chosen": 164.82431030273438,
+      "logits/rejected": 173.40679931640625,
+      "logps/chosen": -365.9583740234375,
+      "logps/rejected": -445.6947326660156,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0274367332458496,
+      "rewards/margins": 1.1871185302734375,
+      "rewards/rejected": -2.214555263519287,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 103.49873893671008,
+      "grad_norm": 99.23172159300925,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": 123.42630767822266,
-      "logits/rejected": 112.1140365600586,
-      "logps/chosen": -401.87945556640625,
-      "logps/rejected": -429.3699645996094,
-      "loss": 0.5472,
+      "logits/chosen": 170.08607482910156,
+      "logits/rejected": 162.03958129882812,
+      "logps/chosen": -414.80340576171875,
+      "logps/rejected": -454.820556640625,
+      "loss": 0.5641,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.4228084087371826,
-      "rewards/margins": 1.1271207332611084,
-      "rewards/rejected": -3.54992938041687,
+      "rewards/chosen": -2.624403715133667,
+      "rewards/margins": 0.9876155853271484,
+      "rewards/rejected": -3.6120193004608154,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 120.25232045055446,
+      "grad_norm": 107.79387973262907,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": 138.51483154296875,
-      "logits/rejected": 142.50746154785156,
-      "logps/chosen": -464.8746643066406,
-      "logps/rejected": -547.7520141601562,
-      "loss": 0.5034,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.2957112789154053,
-      "rewards/margins": 1.2795140743255615,
-      "rewards/rejected": -3.575225353240967,
+      "logits/chosen": 177.14224243164062,
+      "logits/rejected": 174.7544708251953,
+      "logps/chosen": -459.03759765625,
+      "logps/rejected": -512.6439208984375,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.4753570556640625,
+      "rewards/margins": 1.3291194438934326,
+      "rewards/rejected": -4.804476737976074,
       "step": 50
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 60.01877502993502,
+      "grad_norm": 53.02406081573369,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": 131.36839294433594,
-      "logits/rejected": 134.95950317382812,
-      "logps/chosen": -444.547607421875,
-      "logps/rejected": -531.6339721679688,
-      "loss": 0.3025,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -2.315918207168579,
-      "rewards/margins": 2.439805269241333,
-      "rewards/rejected": -4.755723476409912,
+      "logits/chosen": 167.0361328125,
+      "logits/rejected": 171.70101928710938,
+      "logps/chosen": -414.3773498535156,
+      "logps/rejected": -524.4793701171875,
+      "loss": 0.3097,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -3.200485944747925,
+      "rewards/margins": 2.440056085586548,
+      "rewards/rejected": -5.6405415534973145,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 47.79216068377628,
+      "grad_norm": 62.10149124492704,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": 124.45259094238281,
-      "logits/rejected": 126.58074951171875,
-      "logps/chosen": -414.2548828125,
-      "logps/rejected": -518.2446899414062,
-      "loss": 0.1949,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -2.3579883575439453,
-      "rewards/margins": 2.8935599327087402,
-      "rewards/rejected": -5.251548767089844,
+      "logits/chosen": 157.87100219726562,
+      "logits/rejected": 165.49331665039062,
+      "logps/chosen": -453.851318359375,
+      "logps/rejected": -509.77960205078125,
+      "loss": 0.2085,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -3.4425768852233887,
+      "rewards/margins": 2.912635326385498,
+      "rewards/rejected": -6.355212211608887,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 55.81661982856631,
+      "grad_norm": 92.48557214337542,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": 110.53846740722656,
-      "logits/rejected": 127.3997573852539,
-      "logps/chosen": -416.94775390625,
-      "logps/rejected": -536.0177001953125,
-      "loss": 0.1738,
+      "logits/chosen": 155.17611694335938,
+      "logits/rejected": 170.6410369873047,
+      "logps/chosen": -473.2515563964844,
+      "logps/rejected": -549.7650146484375,
+      "loss": 0.187,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": -3.1685705184936523,
-      "rewards/margins": 2.967393159866333,
-      "rewards/rejected": -6.135963439941406,
+      "rewards/chosen": -3.7577788829803467,
+      "rewards/margins": 2.7696285247802734,
+      "rewards/rejected": -6.527407646179199,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 46.35415773419707,
+      "grad_norm": 50.507688635772325,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": 108.8740234375,
-      "logits/rejected": 121.63133239746094,
-      "logps/chosen": -452.612060546875,
-      "logps/rejected": -570.0894775390625,
-      "loss": 0.1585,
+      "logits/chosen": 160.5852508544922,
+      "logits/rejected": 169.7069854736328,
+      "logps/chosen": -444.99725341796875,
+      "logps/rejected": -585.0924072265625,
+      "loss": 0.1675,
       "rewards/accuracies": 0.96875,
-      "rewards/chosen": -3.32800030708313,
-      "rewards/margins": 3.267381191253662,
-      "rewards/rejected": -6.595381259918213,
+      "rewards/chosen": -3.7263991832733154,
+      "rewards/margins": 3.2523887157440186,
+      "rewards/rejected": -6.97878885269165,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 43.55253667946353,
+      "grad_norm": 44.901315947621455,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": 115.157958984375,
-      "logits/rejected": 114.7303695678711,
-      "logps/chosen": -443.5245056152344,
-      "logps/rejected": -538.7801513671875,
-      "loss": 0.1534,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": -3.0830078125,
-      "rewards/margins": 3.0285675525665283,
-      "rewards/rejected": -6.111575126647949,
-      "step": 100
-    },
-    {
-      "epoch": 1.8957345971563981,
-      "eval_logits/chosen": 94.33853912353516,
-      "eval_logits/rejected": 88.46772003173828,
-      "eval_logps/chosen": -442.3636779785156,
-      "eval_logps/rejected": -472.6589660644531,
-      "eval_loss": 0.4630958139896393,
-      "eval_rewards/accuracies": 0.7083333134651184,
-      "eval_rewards/chosen": -3.9814913272857666,
-      "eval_rewards/margins": 1.5538396835327148,
-      "eval_rewards/rejected": -5.5353312492370605,
-      "eval_runtime": 21.3945,
-      "eval_samples_per_second": 35.056,
-      "eval_steps_per_second": 1.122,
+      "logits/chosen": 160.45559692382812,
+      "logits/rejected": 161.6788330078125,
+      "logps/chosen": -444.6620178222656,
+      "logps/rejected": -523.9886474609375,
+      "loss": 0.1723,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -3.935753583908081,
+      "rewards/margins": 3.035583734512329,
+      "rewards/rejected": -6.971337795257568,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.38818228932527393,
-      "train_runtime": 1555.646,
-      "train_samples_per_second": 8.678,
-      "train_steps_per_second": 0.067
+      "train_loss": 0.39618923515081406,
+      "train_runtime": 6217.6844,
+      "train_samples_per_second": 2.171,
+      "train_steps_per_second": 0.017
     }
   ],
   "logging_steps": 10,
@@ -217,7 +201,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36c830e7f828745f5f4993639dca5d4e9350a3782d777acf9e26bd26d79229bd
+oid sha256:72c54496520c00ad2acc4b9f3014bb5fb90f7cc0675061cc92b9c15fbde068ae
 size 6456