yiran-wang3 commited on
Commit
a34306c
1 Parent(s): 1dada7a

End of training

Browse files
README.md ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ license: other
4
+ base_model: yiran-wang3/ds_coder6.7b_rmsprop_iter4
5
+ tags:
6
+ - alignment-handbook
7
+ - generated_from_trainer
8
+ - trl
9
+ - dpo
10
+ datasets:
11
+ - reflection-gen/ds_coder6.7b_rmsprop_iter4_sppo_hard_new_cn_mining_oj_iter4-binarized
12
+ model-index:
13
+ - name: ds_coder6.7b_rmsprop_iter5
14
+ results: []
15
+ ---
16
+
17
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
18
+ should probably proofread and complete it, then remove this comment. -->
19
+
20
+ # ds_coder6.7b_rmsprop_iter5
21
+
22
+ This model is a fine-tuned version of [yiran-wang3/ds_coder6.7b_rmsprop_iter4](https://huggingface.co/yiran-wang3/ds_coder6.7b_rmsprop_iter4) on the reflection-gen/ds_coder6.7b_rmsprop_iter4_sppo_hard_new_cn_mining_oj_iter4-binarized dataset.
23
+
24
+ ## Model description
25
+
26
+ More information needed
27
+
28
+ ## Intended uses & limitations
29
+
30
+ More information needed
31
+
32
+ ## Training and evaluation data
33
+
34
+ More information needed
35
+
36
+ ## Training procedure
37
+
38
+ ### Training hyperparameters
39
+
40
+ The following hyperparameters were used during training:
41
+ - learning_rate: 5e-07
42
+ - train_batch_size: 8
43
+ - eval_batch_size: 4
44
+ - seed: 42
45
+ - distributed_type: multi-GPU
46
+ - num_devices: 8
47
+ - total_train_batch_size: 64
48
+ - total_eval_batch_size: 32
49
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
+ - lr_scheduler_type: constant
51
+ - lr_scheduler_warmup_ratio: 0.1
52
+ - lr_scheduler_warmup_steps: 100
53
+ - num_epochs: 1.0
54
+
55
+ ### Training results
56
+
57
+
58
+
59
+ ### Framework versions
60
+
61
+ - Transformers 4.45.0
62
+ - Pytorch 2.4.0+cu121
63
+ - Datasets 2.14.6
64
+ - Tokenizers 0.20.3
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.4988678726885054,
5
+ "train_runtime": 316.5001,
6
+ "train_samples": 1705,
7
+ "train_samples_per_second": 5.387,
8
+ "train_steps_per_second": 0.085
9
+ }
config.json CHANGED
@@ -29,6 +29,6 @@
29
  "tie_word_embeddings": false,
30
  "torch_dtype": "bfloat16",
31
  "transformers_version": "4.45.0",
32
- "use_cache": false,
33
  "vocab_size": 32256
34
  }
 
29
  "tie_word_embeddings": false,
30
  "torch_dtype": "bfloat16",
31
  "transformers_version": "4.45.0",
32
+ "use_cache": true,
33
  "vocab_size": 32256
34
  }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 32013,
4
+ "eos_token_id": 32021,
5
+ "transformers_version": "4.45.0"
6
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.4988678726885054,
5
+ "train_runtime": 316.5001,
6
+ "train_samples": 1705,
7
+ "train_samples_per_second": 5.387,
8
+ "train_steps_per_second": 0.085
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,609 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 100,
6
+ "global_step": 27,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "debug/policy_chosen_logits": 1.9176132678985596,
13
+ "debug/policy_chosen_logps": -384.8390197753906,
14
+ "debug/policy_rejected_logits": 1.9076852798461914,
15
+ "debug/policy_rejected_logps": -398.07366943359375,
16
+ "debug/reference_chosen_logps": -384.8390197753906,
17
+ "debug/reference_rejected_logps": -398.07366943359375,
18
+ "epoch": 0.037037037037037035,
19
+ "grad_norm": 3.365806843577083,
20
+ "learning_rate": 5e-07,
21
+ "logits/chosen": 1.9176132678985596,
22
+ "logits/rejected": 1.9076852798461914,
23
+ "logps/chosen": -384.8390197753906,
24
+ "logps/rejected": -398.07366943359375,
25
+ "loss": 0.5,
26
+ "rewards/accuracies": 0.0,
27
+ "rewards/chosen": 0.0,
28
+ "rewards/margins": 0.0,
29
+ "rewards/rejected": 0.0,
30
+ "step": 1
31
+ },
32
+ {
33
+ "debug/policy_chosen_logits": 1.5860459804534912,
34
+ "debug/policy_chosen_logps": -391.32177734375,
35
+ "debug/policy_rejected_logits": 1.8992308378219604,
36
+ "debug/policy_rejected_logps": -398.1220397949219,
37
+ "debug/reference_chosen_logps": -392.06561279296875,
38
+ "debug/reference_rejected_logps": -398.2801208496094,
39
+ "epoch": 0.07407407407407407,
40
+ "grad_norm": 3.641806905164277,
41
+ "learning_rate": 5e-07,
42
+ "logits/chosen": 1.5860459804534912,
43
+ "logits/rejected": 1.8992308378219604,
44
+ "logps/chosen": -391.32177734375,
45
+ "logps/rejected": -398.1220397949219,
46
+ "loss": 0.4989,
47
+ "rewards/accuracies": 0.875,
48
+ "rewards/chosen": 0.007438316475600004,
49
+ "rewards/margins": 0.0058576203882694244,
50
+ "rewards/rejected": 0.0015806962037459016,
51
+ "step": 2
52
+ },
53
+ {
54
+ "debug/policy_chosen_logits": 1.9330121278762817,
55
+ "debug/policy_chosen_logps": -374.0859375,
56
+ "debug/policy_rejected_logits": 2.021165609359741,
57
+ "debug/policy_rejected_logps": -400.7498779296875,
58
+ "debug/reference_chosen_logps": -374.1063537597656,
59
+ "debug/reference_rejected_logps": -400.61810302734375,
60
+ "epoch": 0.1111111111111111,
61
+ "grad_norm": 3.138657233492191,
62
+ "learning_rate": 5e-07,
63
+ "logits/chosen": 1.9330121278762817,
64
+ "logits/rejected": 2.021165609359741,
65
+ "logps/chosen": -374.0859375,
66
+ "logps/rejected": -400.7498779296875,
67
+ "loss": 0.4975,
68
+ "rewards/accuracies": 0.5,
69
+ "rewards/chosen": 0.00020408618729561567,
70
+ "rewards/margins": 0.001521835569292307,
71
+ "rewards/rejected": -0.0013177492655813694,
72
+ "step": 3
73
+ },
74
+ {
75
+ "debug/policy_chosen_logits": 2.0667145252227783,
76
+ "debug/policy_chosen_logps": -377.2727355957031,
77
+ "debug/policy_rejected_logits": 2.1134490966796875,
78
+ "debug/policy_rejected_logps": -415.147705078125,
79
+ "debug/reference_chosen_logps": -378.2873229980469,
80
+ "debug/reference_rejected_logps": -416.02130126953125,
81
+ "epoch": 0.14814814814814814,
82
+ "grad_norm": 3.2210878439232937,
83
+ "learning_rate": 5e-07,
84
+ "logits/chosen": 2.0667145252227783,
85
+ "logits/rejected": 2.1134490966796875,
86
+ "logps/chosen": -377.2727355957031,
87
+ "logps/rejected": -415.147705078125,
88
+ "loss": 0.5024,
89
+ "rewards/accuracies": 0.625,
90
+ "rewards/chosen": 0.010145988315343857,
91
+ "rewards/margins": 0.001409988384693861,
92
+ "rewards/rejected": 0.008735999464988708,
93
+ "step": 4
94
+ },
95
+ {
96
+ "debug/policy_chosen_logits": 2.04209303855896,
97
+ "debug/policy_chosen_logps": -389.889892578125,
98
+ "debug/policy_rejected_logits": 1.9708077907562256,
99
+ "debug/policy_rejected_logps": -386.6638488769531,
100
+ "debug/reference_chosen_logps": -387.9091796875,
101
+ "debug/reference_rejected_logps": -384.6646728515625,
102
+ "epoch": 0.18518518518518517,
103
+ "grad_norm": 3.3211093740721696,
104
+ "learning_rate": 5e-07,
105
+ "logits/chosen": 2.04209303855896,
106
+ "logits/rejected": 1.9708077907562256,
107
+ "logps/chosen": -389.889892578125,
108
+ "logps/rejected": -386.6638488769531,
109
+ "loss": 0.4995,
110
+ "rewards/accuracies": 0.5,
111
+ "rewards/chosen": -0.01980709098279476,
112
+ "rewards/margins": 0.0001847841776907444,
113
+ "rewards/rejected": -0.01999187469482422,
114
+ "step": 5
115
+ },
116
+ {
117
+ "debug/policy_chosen_logits": 2.0924808979034424,
118
+ "debug/policy_chosen_logps": -396.0131530761719,
119
+ "debug/policy_rejected_logits": 1.9733433723449707,
120
+ "debug/policy_rejected_logps": -401.1158752441406,
121
+ "debug/reference_chosen_logps": -399.01123046875,
122
+ "debug/reference_rejected_logps": -403.74298095703125,
123
+ "epoch": 0.2222222222222222,
124
+ "grad_norm": 4.093512592438682,
125
+ "learning_rate": 5e-07,
126
+ "logits/chosen": 2.0924808979034424,
127
+ "logits/rejected": 1.9733433723449707,
128
+ "logps/chosen": -396.0131530761719,
129
+ "logps/rejected": -401.1158752441406,
130
+ "loss": 0.4983,
131
+ "rewards/accuracies": 0.5,
132
+ "rewards/chosen": 0.029980963096022606,
133
+ "rewards/margins": 0.003709678538143635,
134
+ "rewards/rejected": 0.026271285489201546,
135
+ "step": 6
136
+ },
137
+ {
138
+ "debug/policy_chosen_logits": 1.9140172004699707,
139
+ "debug/policy_chosen_logps": -409.95819091796875,
140
+ "debug/policy_rejected_logits": 1.723309874534607,
141
+ "debug/policy_rejected_logps": -395.7958984375,
142
+ "debug/reference_chosen_logps": -407.73455810546875,
143
+ "debug/reference_rejected_logps": -393.78973388671875,
144
+ "epoch": 0.25925925925925924,
145
+ "grad_norm": 4.141926483884428,
146
+ "learning_rate": 5e-07,
147
+ "logits/chosen": 1.9140172004699707,
148
+ "logits/rejected": 1.723309874534607,
149
+ "logps/chosen": -409.95819091796875,
150
+ "logps/rejected": -395.7958984375,
151
+ "loss": 0.4981,
152
+ "rewards/accuracies": 0.375,
153
+ "rewards/chosen": -0.02223609760403633,
154
+ "rewards/margins": -0.002174415159970522,
155
+ "rewards/rejected": -0.020061682909727097,
156
+ "step": 7
157
+ },
158
+ {
159
+ "debug/policy_chosen_logits": 1.6491832733154297,
160
+ "debug/policy_chosen_logps": -378.5914306640625,
161
+ "debug/policy_rejected_logits": 1.6308629512786865,
162
+ "debug/policy_rejected_logps": -388.767822265625,
163
+ "debug/reference_chosen_logps": -379.73565673828125,
164
+ "debug/reference_rejected_logps": -389.461669921875,
165
+ "epoch": 0.2962962962962963,
166
+ "grad_norm": 3.235474686776188,
167
+ "learning_rate": 5e-07,
168
+ "logits/chosen": 1.6491832733154297,
169
+ "logits/rejected": 1.6308629512786865,
170
+ "logps/chosen": -378.5914306640625,
171
+ "logps/rejected": -388.767822265625,
172
+ "loss": 0.4972,
173
+ "rewards/accuracies": 0.5,
174
+ "rewards/chosen": 0.01144229806959629,
175
+ "rewards/margins": 0.004504012875258923,
176
+ "rewards/rejected": 0.006938285659998655,
177
+ "step": 8
178
+ },
179
+ {
180
+ "debug/policy_chosen_logits": 1.983660340309143,
181
+ "debug/policy_chosen_logps": -385.3446350097656,
182
+ "debug/policy_rejected_logits": 1.91273832321167,
183
+ "debug/policy_rejected_logps": -392.86297607421875,
184
+ "debug/reference_chosen_logps": -384.8369445800781,
185
+ "debug/reference_rejected_logps": -391.40753173828125,
186
+ "epoch": 0.3333333333333333,
187
+ "grad_norm": 2.9856408713590152,
188
+ "learning_rate": 5e-07,
189
+ "logits/chosen": 1.983660340309143,
190
+ "logits/rejected": 1.91273832321167,
191
+ "logps/chosen": -385.3446350097656,
192
+ "logps/rejected": -392.86297607421875,
193
+ "loss": 0.4959,
194
+ "rewards/accuracies": 0.75,
195
+ "rewards/chosen": -0.0050768665969371796,
196
+ "rewards/margins": 0.009477272629737854,
197
+ "rewards/rejected": -0.014554138295352459,
198
+ "step": 9
199
+ },
200
+ {
201
+ "debug/policy_chosen_logits": 1.7573912143707275,
202
+ "debug/policy_chosen_logps": -405.07745361328125,
203
+ "debug/policy_rejected_logits": 1.9942240715026855,
204
+ "debug/policy_rejected_logps": -410.64959716796875,
205
+ "debug/reference_chosen_logps": -406.0697021484375,
206
+ "debug/reference_rejected_logps": -412.43084716796875,
207
+ "epoch": 0.37037037037037035,
208
+ "grad_norm": 4.440636208350787,
209
+ "learning_rate": 5e-07,
210
+ "logits/chosen": 1.7573912143707275,
211
+ "logits/rejected": 1.9942240715026855,
212
+ "logps/chosen": -405.07745361328125,
213
+ "logps/rejected": -410.64959716796875,
214
+ "loss": 0.4989,
215
+ "rewards/accuracies": 0.25,
216
+ "rewards/chosen": 0.009922904893755913,
217
+ "rewards/margins": -0.007889671251177788,
218
+ "rewards/rejected": 0.0178125761449337,
219
+ "step": 10
220
+ },
221
+ {
222
+ "debug/policy_chosen_logits": 1.6106464862823486,
223
+ "debug/policy_chosen_logps": -397.54608154296875,
224
+ "debug/policy_rejected_logits": 1.8594430685043335,
225
+ "debug/policy_rejected_logps": -398.8537902832031,
226
+ "debug/reference_chosen_logps": -397.4752197265625,
227
+ "debug/reference_rejected_logps": -398.4106750488281,
228
+ "epoch": 0.4074074074074074,
229
+ "grad_norm": 3.93748886341799,
230
+ "learning_rate": 5e-07,
231
+ "logits/chosen": 1.6106464862823486,
232
+ "logits/rejected": 1.8594430685043335,
233
+ "logps/chosen": -397.54608154296875,
234
+ "logps/rejected": -398.8537902832031,
235
+ "loss": 0.4958,
236
+ "rewards/accuracies": 0.5,
237
+ "rewards/chosen": -0.0007085420656949282,
238
+ "rewards/margins": 0.0037224958650767803,
239
+ "rewards/rejected": -0.004431037697941065,
240
+ "step": 11
241
+ },
242
+ {
243
+ "debug/policy_chosen_logits": 1.9425033330917358,
244
+ "debug/policy_chosen_logps": -386.7259521484375,
245
+ "debug/policy_rejected_logits": 1.8076810836791992,
246
+ "debug/policy_rejected_logps": -409.758056640625,
247
+ "debug/reference_chosen_logps": -386.6396484375,
248
+ "debug/reference_rejected_logps": -409.6361999511719,
249
+ "epoch": 0.4444444444444444,
250
+ "grad_norm": 2.9994704461485444,
251
+ "learning_rate": 5e-07,
252
+ "logits/chosen": 1.9425033330917358,
253
+ "logits/rejected": 1.8076810836791992,
254
+ "logps/chosen": -386.7259521484375,
255
+ "logps/rejected": -409.758056640625,
256
+ "loss": 0.4957,
257
+ "rewards/accuracies": 0.375,
258
+ "rewards/chosen": -0.0008631134405732155,
259
+ "rewards/margins": 0.0003554532304406166,
260
+ "rewards/rejected": -0.0012185671366751194,
261
+ "step": 12
262
+ },
263
+ {
264
+ "debug/policy_chosen_logits": 1.980305552482605,
265
+ "debug/policy_chosen_logps": -403.00634765625,
266
+ "debug/policy_rejected_logits": 2.0576224327087402,
267
+ "debug/policy_rejected_logps": -399.4578857421875,
268
+ "debug/reference_chosen_logps": -402.3633117675781,
269
+ "debug/reference_rejected_logps": -399.6813659667969,
270
+ "epoch": 0.48148148148148145,
271
+ "grad_norm": 2.8730835745151624,
272
+ "learning_rate": 5e-07,
273
+ "logits/chosen": 1.980305552482605,
274
+ "logits/rejected": 2.0576224327087402,
275
+ "logps/chosen": -403.00634765625,
276
+ "logps/rejected": -399.4578857421875,
277
+ "loss": 0.4998,
278
+ "rewards/accuracies": 0.25,
279
+ "rewards/chosen": -0.006430511362850666,
280
+ "rewards/margins": -0.008665123023092747,
281
+ "rewards/rejected": 0.0022346116602420807,
282
+ "step": 13
283
+ },
284
+ {
285
+ "debug/policy_chosen_logits": 1.746868371963501,
286
+ "debug/policy_chosen_logps": -380.3934326171875,
287
+ "debug/policy_rejected_logits": 1.8052778244018555,
288
+ "debug/policy_rejected_logps": -397.5988464355469,
289
+ "debug/reference_chosen_logps": -380.31329345703125,
290
+ "debug/reference_rejected_logps": -397.5308837890625,
291
+ "epoch": 0.5185185185185185,
292
+ "grad_norm": 2.9568052002579495,
293
+ "learning_rate": 5e-07,
294
+ "logits/chosen": 1.746868371963501,
295
+ "logits/rejected": 1.8052778244018555,
296
+ "logps/chosen": -380.3934326171875,
297
+ "logps/rejected": -397.5988464355469,
298
+ "loss": 0.5027,
299
+ "rewards/accuracies": 0.25,
300
+ "rewards/chosen": -0.0008013155311346054,
301
+ "rewards/margins": -0.00012214668095111847,
302
+ "rewards/rejected": -0.0006791686173528433,
303
+ "step": 14
304
+ },
305
+ {
306
+ "debug/policy_chosen_logits": 1.8233588933944702,
307
+ "debug/policy_chosen_logps": -379.78424072265625,
308
+ "debug/policy_rejected_logits": 1.5954158306121826,
309
+ "debug/policy_rejected_logps": -382.91845703125,
310
+ "debug/reference_chosen_logps": -379.1283264160156,
311
+ "debug/reference_rejected_logps": -381.09710693359375,
312
+ "epoch": 0.5555555555555556,
313
+ "grad_norm": 3.235492786450746,
314
+ "learning_rate": 5e-07,
315
+ "logits/chosen": 1.8233588933944702,
316
+ "logits/rejected": 1.5954158306121826,
317
+ "logps/chosen": -379.78424072265625,
318
+ "logps/rejected": -382.91845703125,
319
+ "loss": 0.499,
320
+ "rewards/accuracies": 0.75,
321
+ "rewards/chosen": -0.006559104658663273,
322
+ "rewards/margins": 0.011654319241642952,
323
+ "rewards/rejected": -0.0182134248316288,
324
+ "step": 15
325
+ },
326
+ {
327
+ "debug/policy_chosen_logits": 1.9640549421310425,
328
+ "debug/policy_chosen_logps": -394.0776672363281,
329
+ "debug/policy_rejected_logits": 2.2034473419189453,
330
+ "debug/policy_rejected_logps": -415.4656066894531,
331
+ "debug/reference_chosen_logps": -396.9149169921875,
332
+ "debug/reference_rejected_logps": -418.02545166015625,
333
+ "epoch": 0.5925925925925926,
334
+ "grad_norm": 3.4387048010523635,
335
+ "learning_rate": 5e-07,
336
+ "logits/chosen": 1.9640549421310425,
337
+ "logits/rejected": 2.2034473419189453,
338
+ "logps/chosen": -394.0776672363281,
339
+ "logps/rejected": -415.4656066894531,
340
+ "loss": 0.494,
341
+ "rewards/accuracies": 0.5,
342
+ "rewards/chosen": 0.028372764587402344,
343
+ "rewards/margins": 0.0027741624508053064,
344
+ "rewards/rejected": 0.02559860236942768,
345
+ "step": 16
346
+ },
347
+ {
348
+ "debug/policy_chosen_logits": 1.7394020557403564,
349
+ "debug/policy_chosen_logps": -392.60986328125,
350
+ "debug/policy_rejected_logits": 2.068730115890503,
351
+ "debug/policy_rejected_logps": -384.3857421875,
352
+ "debug/reference_chosen_logps": -393.29168701171875,
353
+ "debug/reference_rejected_logps": -383.3329772949219,
354
+ "epoch": 0.6296296296296297,
355
+ "grad_norm": 2.934128028135007,
356
+ "learning_rate": 5e-07,
357
+ "logits/chosen": 1.7394020557403564,
358
+ "logits/rejected": 2.068730115890503,
359
+ "logps/chosen": -392.60986328125,
360
+ "logps/rejected": -384.3857421875,
361
+ "loss": 0.4975,
362
+ "rewards/accuracies": 1.0,
363
+ "rewards/chosen": 0.006818465888500214,
364
+ "rewards/margins": 0.0173458494246006,
365
+ "rewards/rejected": -0.010527381673455238,
366
+ "step": 17
367
+ },
368
+ {
369
+ "debug/policy_chosen_logits": 1.707554578781128,
370
+ "debug/policy_chosen_logps": -405.7987365722656,
371
+ "debug/policy_rejected_logits": 1.5863378047943115,
372
+ "debug/policy_rejected_logps": -381.5325012207031,
373
+ "debug/reference_chosen_logps": -405.7820739746094,
374
+ "debug/reference_rejected_logps": -381.1623229980469,
375
+ "epoch": 0.6666666666666666,
376
+ "grad_norm": 3.304847917225049,
377
+ "learning_rate": 5e-07,
378
+ "logits/chosen": 1.707554578781128,
379
+ "logits/rejected": 1.5863378047943115,
380
+ "logps/chosen": -405.7987365722656,
381
+ "logps/rejected": -381.5325012207031,
382
+ "loss": 0.4993,
383
+ "rewards/accuracies": 0.625,
384
+ "rewards/chosen": -0.00016651151236146688,
385
+ "rewards/margins": 0.003535156138241291,
386
+ "rewards/rejected": -0.0037016679998487234,
387
+ "step": 18
388
+ },
389
+ {
390
+ "debug/policy_chosen_logits": 1.69636070728302,
391
+ "debug/policy_chosen_logps": -377.90325927734375,
392
+ "debug/policy_rejected_logits": 1.7989388704299927,
393
+ "debug/policy_rejected_logps": -386.04559326171875,
394
+ "debug/reference_chosen_logps": -377.9969482421875,
395
+ "debug/reference_rejected_logps": -386.39825439453125,
396
+ "epoch": 0.7037037037037037,
397
+ "grad_norm": 3.0806729050366926,
398
+ "learning_rate": 5e-07,
399
+ "logits/chosen": 1.69636070728302,
400
+ "logits/rejected": 1.7989388704299927,
401
+ "logps/chosen": -377.90325927734375,
402
+ "logps/rejected": -386.04559326171875,
403
+ "loss": 0.4986,
404
+ "rewards/accuracies": 0.375,
405
+ "rewards/chosen": 0.0009368133614771068,
406
+ "rewards/margins": -0.002589988522231579,
407
+ "rewards/rejected": 0.003526801709085703,
408
+ "step": 19
409
+ },
410
+ {
411
+ "debug/policy_chosen_logits": 1.9408522844314575,
412
+ "debug/policy_chosen_logps": -389.70721435546875,
413
+ "debug/policy_rejected_logits": 1.9151865243911743,
414
+ "debug/policy_rejected_logps": -414.2107238769531,
415
+ "debug/reference_chosen_logps": -388.886474609375,
416
+ "debug/reference_rejected_logps": -413.4195556640625,
417
+ "epoch": 0.7407407407407407,
418
+ "grad_norm": 3.0010377273894977,
419
+ "learning_rate": 5e-07,
420
+ "logits/chosen": 1.9408522844314575,
421
+ "logits/rejected": 1.9151865243911743,
422
+ "logps/chosen": -389.70721435546875,
423
+ "logps/rejected": -414.2107238769531,
424
+ "loss": 0.4998,
425
+ "rewards/accuracies": 0.375,
426
+ "rewards/chosen": -0.008207131177186966,
427
+ "rewards/margins": -0.0002954102819785476,
428
+ "rewards/rejected": -0.007911719381809235,
429
+ "step": 20
430
+ },
431
+ {
432
+ "debug/policy_chosen_logits": 2.1346023082733154,
433
+ "debug/policy_chosen_logps": -375.6856994628906,
434
+ "debug/policy_rejected_logits": 2.0008819103240967,
435
+ "debug/policy_rejected_logps": -394.83551025390625,
436
+ "debug/reference_chosen_logps": -375.5802001953125,
437
+ "debug/reference_rejected_logps": -394.8753662109375,
438
+ "epoch": 0.7777777777777778,
439
+ "grad_norm": 3.128585405255993,
440
+ "learning_rate": 5e-07,
441
+ "logits/chosen": 2.1346023082733154,
442
+ "logits/rejected": 2.0008819103240967,
443
+ "logps/chosen": -375.6856994628906,
444
+ "logps/rejected": -394.83551025390625,
445
+ "loss": 0.4957,
446
+ "rewards/accuracies": 0.375,
447
+ "rewards/chosen": -0.0010547256097197533,
448
+ "rewards/margins": -0.0014535905793309212,
449
+ "rewards/rejected": 0.00039886473678052425,
450
+ "step": 21
451
+ },
452
+ {
453
+ "debug/policy_chosen_logits": 2.0572948455810547,
454
+ "debug/policy_chosen_logps": -388.89495849609375,
455
+ "debug/policy_rejected_logits": 2.174546957015991,
456
+ "debug/policy_rejected_logps": -411.72747802734375,
457
+ "debug/reference_chosen_logps": -389.1768798828125,
458
+ "debug/reference_rejected_logps": -410.6370544433594,
459
+ "epoch": 0.8148148148148148,
460
+ "grad_norm": 2.9815920689778626,
461
+ "learning_rate": 5e-07,
462
+ "logits/chosen": 2.0572948455810547,
463
+ "logits/rejected": 2.174546957015991,
464
+ "logps/chosen": -388.89495849609375,
465
+ "logps/rejected": -411.72747802734375,
466
+ "loss": 0.5,
467
+ "rewards/accuracies": 0.75,
468
+ "rewards/chosen": 0.0028189467266201973,
469
+ "rewards/margins": 0.013723219744861126,
470
+ "rewards/rejected": -0.010904273018240929,
471
+ "step": 22
472
+ },
473
+ {
474
+ "debug/policy_chosen_logits": 1.9259278774261475,
475
+ "debug/policy_chosen_logps": -369.13165283203125,
476
+ "debug/policy_rejected_logits": 2.0857815742492676,
477
+ "debug/policy_rejected_logps": -386.7788391113281,
478
+ "debug/reference_chosen_logps": -369.52679443359375,
479
+ "debug/reference_rejected_logps": -386.36932373046875,
480
+ "epoch": 0.8518518518518519,
481
+ "grad_norm": 2.9271229511243977,
482
+ "learning_rate": 5e-07,
483
+ "logits/chosen": 1.9259278774261475,
484
+ "logits/rejected": 2.0857815742492676,
485
+ "logps/chosen": -369.13165283203125,
486
+ "logps/rejected": -386.7788391113281,
487
+ "loss": 0.4964,
488
+ "rewards/accuracies": 0.5,
489
+ "rewards/chosen": 0.003951530437916517,
490
+ "rewards/margins": 0.00804698932915926,
491
+ "rewards/rejected": -0.0040954588912427425,
492
+ "step": 23
493
+ },
494
+ {
495
+ "debug/policy_chosen_logits": 1.8484477996826172,
496
+ "debug/policy_chosen_logps": -394.09912109375,
497
+ "debug/policy_rejected_logits": 2.0279951095581055,
498
+ "debug/policy_rejected_logps": -395.52679443359375,
499
+ "debug/reference_chosen_logps": -395.26434326171875,
500
+ "debug/reference_rejected_logps": -396.5709228515625,
501
+ "epoch": 0.8888888888888888,
502
+ "grad_norm": 3.061452537487587,
503
+ "learning_rate": 5e-07,
504
+ "logits/chosen": 1.8484477996826172,
505
+ "logits/rejected": 2.0279951095581055,
506
+ "logps/chosen": -394.09912109375,
507
+ "logps/rejected": -395.52679443359375,
508
+ "loss": 0.4996,
509
+ "rewards/accuracies": 0.5,
510
+ "rewards/chosen": 0.011652068234980106,
511
+ "rewards/margins": 0.0012104416964575648,
512
+ "rewards/rejected": 0.010441627353429794,
513
+ "step": 24
514
+ },
515
+ {
516
+ "debug/policy_chosen_logits": 1.6762018203735352,
517
+ "debug/policy_chosen_logps": -354.19732666015625,
518
+ "debug/policy_rejected_logits": 1.7495834827423096,
519
+ "debug/policy_rejected_logps": -370.53057861328125,
520
+ "debug/reference_chosen_logps": -354.2095947265625,
521
+ "debug/reference_rejected_logps": -369.8760681152344,
522
+ "epoch": 0.9259259259259259,
523
+ "grad_norm": 3.239967073927591,
524
+ "learning_rate": 5e-07,
525
+ "logits/chosen": 1.6762018203735352,
526
+ "logits/rejected": 1.7495834827423096,
527
+ "logps/chosen": -354.19732666015625,
528
+ "logps/rejected": -370.53057861328125,
529
+ "loss": 0.4973,
530
+ "rewards/accuracies": 0.75,
531
+ "rewards/chosen": 0.00012287136632949114,
532
+ "rewards/margins": 0.006667976267635822,
533
+ "rewards/rejected": -0.006545105017721653,
534
+ "step": 25
535
+ },
536
+ {
537
+ "debug/policy_chosen_logits": 1.5777896642684937,
538
+ "debug/policy_chosen_logps": -390.37152099609375,
539
+ "debug/policy_rejected_logits": 1.8194000720977783,
540
+ "debug/policy_rejected_logps": -395.33477783203125,
541
+ "debug/reference_chosen_logps": -390.51409912109375,
542
+ "debug/reference_rejected_logps": -395.2436218261719,
543
+ "epoch": 0.9629629629629629,
544
+ "grad_norm": 3.267396833604278,
545
+ "learning_rate": 5e-07,
546
+ "logits/chosen": 1.5777896642684937,
547
+ "logits/rejected": 1.8194000720977783,
548
+ "logps/chosen": -390.37152099609375,
549
+ "logps/rejected": -395.33477783203125,
550
+ "loss": 0.4993,
551
+ "rewards/accuracies": 0.5,
552
+ "rewards/chosen": 0.001425781287252903,
553
+ "rewards/margins": 0.0023374559823423624,
554
+ "rewards/rejected": -0.0009116745204664767,
555
+ "step": 26
556
+ },
557
+ {
558
+ "debug/policy_chosen_logits": 1.8320817947387695,
559
+ "debug/policy_chosen_logps": -395.1331787109375,
560
+ "debug/policy_rejected_logits": 1.9883564710617065,
561
+ "debug/policy_rejected_logps": -386.84326171875,
562
+ "debug/reference_chosen_logps": -394.580810546875,
563
+ "debug/reference_rejected_logps": -385.752685546875,
564
+ "epoch": 1.0,
565
+ "grad_norm": 3.078555101079866,
566
+ "learning_rate": 5e-07,
567
+ "logits/chosen": 1.8320817947387695,
568
+ "logits/rejected": 1.9883564710617065,
569
+ "logps/chosen": -395.1331787109375,
570
+ "logps/rejected": -386.84326171875,
571
+ "loss": 0.5125,
572
+ "rewards/accuracies": 0.625,
573
+ "rewards/chosen": -0.005523948930203915,
574
+ "rewards/margins": 0.00538150779902935,
575
+ "rewards/rejected": -0.01090545579791069,
576
+ "step": 27
577
+ },
578
+ {
579
+ "epoch": 1.0,
580
+ "step": 27,
581
+ "total_flos": 0.0,
582
+ "train_loss": 0.4988678726885054,
583
+ "train_runtime": 316.5001,
584
+ "train_samples_per_second": 5.387,
585
+ "train_steps_per_second": 0.085
586
+ }
587
+ ],
588
+ "logging_steps": 1,
589
+ "max_steps": 27,
590
+ "num_input_tokens_seen": 0,
591
+ "num_train_epochs": 1,
592
+ "save_steps": 500,
593
+ "stateful_callbacks": {
594
+ "TrainerControl": {
595
+ "args": {
596
+ "should_epoch_stop": false,
597
+ "should_evaluate": false,
598
+ "should_log": false,
599
+ "should_save": true,
600
+ "should_training_stop": true
601
+ },
602
+ "attributes": {}
603
+ }
604
+ },
605
+ "total_flos": 0.0,
606
+ "train_batch_size": 8,
607
+ "trial_name": null,
608
+ "trial_params": null
609
+ }