Model save

README.md

@@ -0,0 +1,72 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/mistral-7b-sft-constitutional-ai
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: mistral-7b-dpo-constitutional-ai
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-7b-dpo-constitutional-ai
+
+This model is a fine-tuned version of [alignment-handbook/mistral-7b-sft-constitutional-ai](https://huggingface.co/alignment-handbook/mistral-7b-sft-constitutional-ai) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5467
+- Rewards/chosen: -3.4828
+- Rewards/rejected: -6.0365
+- Rewards/accuracies: 0.7025
+- Rewards/margins: 2.5536
+- Logps/rejected: -230.1307
+- Logps/chosen: -225.5250
+- Logits/rejected: -2.9599
+- Logits/chosen: -2.9763
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 16
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.0884        | 1.94  | 1000 | 0.5467          | -3.4828        | -6.0365          | 0.7025             | 2.5536          | -230.1307      | -225.5250    | -2.9599         | -2.9763       |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.0a0+32f93b1
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.25373340110176973,
+    "train_runtime": 3870.3099,
+    "train_samples": 8239,
+    "train_samples_per_second": 6.386,
+    "train_steps_per_second": 0.399
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:555b4c77f00b69c555707be1cfda0f060a91212a7bf9fa8113adbb31d92b6468
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a7a9d8d005a34735f380a9df4d119f75f209a1a751c4784037bc8856998521
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cec88ff33d8d7f1750dfc6de76ff4547859d107afbc851a29abd44c78dc79bde
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Apr10_00-26-56_gx-dev-master-0/events.out.tfevents.1712708949.gx-dev-master-0.951.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:920b5eab3e16d9f81f1f1b7ffafb80be53bb04d70cad6b18cac07724289c4aba
-size 74830
+oid sha256:b0b48abd7abb00f6dda39b021cd0d1de58b73e0a1aa457630d1c9fe448adffd0
+size 112336

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.25373340110176973,
+    "train_runtime": 3870.3099,
+    "train_samples": 8239,
+    "train_samples_per_second": 6.386,
+    "train_steps_per_second": 0.399
+}

trainer_state.json

@@ -0,0 +1,2371 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 1000,
+  "global_step": 1545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 238.7771549579817,
+      "learning_rate": 3.225806451612903e-09,
+      "logits/chosen": -3.3442530632019043,
+      "logits/rejected": -3.478727102279663,
+      "logps/chosen": -194.86203002929688,
+      "logps/rejected": -182.01235961914062,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 210.0539707320501,
+      "learning_rate": 3.225806451612903e-08,
+      "logits/chosen": -3.15413236618042,
+      "logits/rejected": -3.161997079849243,
+      "logps/chosen": -265.6070556640625,
+      "logps/rejected": -222.88095092773438,
+      "loss": 0.6949,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.011013220064342022,
+      "rewards/margins": 0.011229809373617172,
+      "rewards/rejected": -0.00021659024059772491,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 176.7933427886698,
+      "learning_rate": 6.451612903225806e-08,
+      "logits/chosen": -3.0983855724334717,
+      "logits/rejected": -3.1877849102020264,
+      "logps/chosen": -166.89926147460938,
+      "logps/rejected": -166.43698120117188,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.009564275853335857,
+      "rewards/margins": 0.0018194920849055052,
+      "rewards/rejected": 0.007744784001260996,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 203.6488787858732,
+      "learning_rate": 9.677419354838709e-08,
+      "logits/chosen": -3.0741190910339355,
+      "logits/rejected": -3.1300857067108154,
+      "logps/chosen": -228.1751251220703,
+      "logps/rejected": -237.1351776123047,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.030515898019075394,
+      "rewards/margins": 0.031470950692892075,
+      "rewards/rejected": -0.000955055293161422,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 193.58974634035104,
+      "learning_rate": 1.2903225806451611e-07,
+      "logits/chosen": -3.169820785522461,
+      "logits/rejected": -3.0861575603485107,
+      "logps/chosen": -221.1776885986328,
+      "logps/rejected": -159.6044464111328,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.06581683456897736,
+      "rewards/margins": 0.07549744844436646,
+      "rewards/rejected": -0.009680609218776226,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 204.80430949892167,
+      "learning_rate": 1.6129032258064515e-07,
+      "logits/chosen": -3.061490535736084,
+      "logits/rejected": -3.091951847076416,
+      "logps/chosen": -189.7487335205078,
+      "logps/rejected": -243.40603637695312,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.1369192898273468,
+      "rewards/margins": 0.06373085081577301,
+      "rewards/rejected": 0.07318843901157379,
+      "step": 50
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 184.5945707092153,
+      "learning_rate": 1.9354838709677418e-07,
+      "logits/chosen": -3.197416305541992,
+      "logits/rejected": -3.229931592941284,
+      "logps/chosen": -258.2798156738281,
+      "logps/rejected": -230.84716796875,
+      "loss": 0.6548,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1522468477487564,
+      "rewards/margins": 0.250114768743515,
+      "rewards/rejected": -0.0978679284453392,
+      "step": 60
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 186.27179515709204,
+      "learning_rate": 2.2580645161290322e-07,
+      "logits/chosen": -2.967573404312134,
+      "logits/rejected": -3.0102295875549316,
+      "logps/chosen": -224.0232391357422,
+      "logps/rejected": -250.5163116455078,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.29443851113319397,
+      "rewards/margins": 0.16650478541851044,
+      "rewards/rejected": 0.12793374061584473,
+      "step": 70
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 270.5970909334128,
+      "learning_rate": 2.5806451612903223e-07,
+      "logits/chosen": -3.2193164825439453,
+      "logits/rejected": -3.1845672130584717,
+      "logps/chosen": -215.4364776611328,
+      "logps/rejected": -197.53713989257812,
+      "loss": 0.6466,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.18258078396320343,
+      "rewards/margins": 0.08748456835746765,
+      "rewards/rejected": 0.09509620815515518,
+      "step": 80
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 205.49332887939408,
+      "learning_rate": 2.903225806451613e-07,
+      "logits/chosen": -3.09409499168396,
+      "logits/rejected": -3.0983214378356934,
+      "logps/chosen": -103.38987731933594,
+      "logps/rejected": -143.33724975585938,
+      "loss": 0.6189,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.08013658225536346,
+      "rewards/margins": 0.021646564826369286,
+      "rewards/rejected": 0.05849001556634903,
+      "step": 90
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 125.49552783164246,
+      "learning_rate": 3.225806451612903e-07,
+      "logits/chosen": -3.0359697341918945,
+      "logits/rejected": -3.104940891265869,
+      "logps/chosen": -159.39813232421875,
+      "logps/rejected": -180.9935302734375,
+      "loss": 0.6091,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.13538669049739838,
+      "rewards/margins": 0.23133841156959534,
+      "rewards/rejected": -0.09595172107219696,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 220.0417527459765,
+      "learning_rate": 3.5483870967741936e-07,
+      "logits/chosen": -3.001208782196045,
+      "logits/rejected": -2.9704670906066895,
+      "logps/chosen": -183.01312255859375,
+      "logps/rejected": -209.7218475341797,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.03632747381925583,
+      "rewards/margins": 0.1413092315196991,
+      "rewards/rejected": -0.17763671278953552,
+      "step": 110
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 168.84413823172952,
+      "learning_rate": 3.8709677419354837e-07,
+      "logits/chosen": -3.1182332038879395,
+      "logits/rejected": -3.0886902809143066,
+      "logps/chosen": -189.5988311767578,
+      "logps/rejected": -178.51364135742188,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.06012638285756111,
+      "rewards/margins": 0.12087175995111465,
+      "rewards/rejected": -0.18099813163280487,
+      "step": 120
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 208.2575863637529,
+      "learning_rate": 4.1935483870967743e-07,
+      "logits/chosen": -3.1611361503601074,
+      "logits/rejected": -3.1497890949249268,
+      "logps/chosen": -157.99807739257812,
+      "logps/rejected": -189.0376739501953,
+      "loss": 0.532,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.09179370105266571,
+      "rewards/margins": 0.6432317495346069,
+      "rewards/rejected": -0.55143803358078,
+      "step": 130
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 150.85083560407247,
+      "learning_rate": 4.5161290322580644e-07,
+      "logits/chosen": -3.091956377029419,
+      "logits/rejected": -3.1225342750549316,
+      "logps/chosen": -185.01499938964844,
+      "logps/rejected": -240.418212890625,
+      "loss": 0.567,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.088681660592556,
+      "rewards/margins": 0.8317887187004089,
+      "rewards/rejected": -0.7431070804595947,
+      "step": 140
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 208.61538011383496,
+      "learning_rate": 4.838709677419355e-07,
+      "logits/chosen": -3.1025145053863525,
+      "logits/rejected": -3.107062578201294,
+      "logps/chosen": -242.4476318359375,
+      "logps/rejected": -186.19691467285156,
+      "loss": 0.5551,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.16192469000816345,
+      "rewards/margins": 0.43396979570388794,
+      "rewards/rejected": -0.2720451056957245,
+      "step": 150
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 227.4405540805906,
+      "learning_rate": 4.982014388489209e-07,
+      "logits/chosen": -3.05570650100708,
+      "logits/rejected": -3.053194761276245,
+      "logps/chosen": -243.532958984375,
+      "logps/rejected": -238.6374053955078,
+      "loss": 0.5381,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.14193885028362274,
+      "rewards/margins": 1.0523029565811157,
+      "rewards/rejected": -1.1942418813705444,
+      "step": 160
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 185.31205838070358,
+      "learning_rate": 4.946043165467625e-07,
+      "logits/chosen": -3.031973361968994,
+      "logits/rejected": -2.987246036529541,
+      "logps/chosen": -148.59695434570312,
+      "logps/rejected": -111.39070892333984,
+      "loss": 0.5769,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12481436878442764,
+      "rewards/margins": 0.8159207105636597,
+      "rewards/rejected": -0.9407350420951843,
+      "step": 170
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 228.1100532242365,
+      "learning_rate": 4.910071942446043e-07,
+      "logits/chosen": -3.133244276046753,
+      "logits/rejected": -3.033322334289551,
+      "logps/chosen": -329.6561279296875,
+      "logps/rejected": -191.4559783935547,
+      "loss": 0.5142,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.49908939003944397,
+      "rewards/margins": 1.1547267436981201,
+      "rewards/rejected": -0.6556375026702881,
+      "step": 180
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 215.88646054369426,
+      "learning_rate": 4.87410071942446e-07,
+      "logits/chosen": -3.1060004234313965,
+      "logits/rejected": -3.1299846172332764,
+      "logps/chosen": -234.3701629638672,
+      "logps/rejected": -184.80455017089844,
+      "loss": 0.5214,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3190203309059143,
+      "rewards/margins": 0.7846688032150269,
+      "rewards/rejected": -1.103689193725586,
+      "step": 190
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 96.46619144056267,
+      "learning_rate": 4.838129496402878e-07,
+      "logits/chosen": -3.0408408641815186,
+      "logits/rejected": -3.1078829765319824,
+      "logps/chosen": -197.76657104492188,
+      "logps/rejected": -256.5730285644531,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.9596657752990723,
+      "rewards/margins": 1.5147722959518433,
+      "rewards/rejected": -0.555106520652771,
+      "step": 200
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 176.7140571201679,
+      "learning_rate": 4.802158273381295e-07,
+      "logits/chosen": -3.130012035369873,
+      "logits/rejected": -2.9814813137054443,
+      "logps/chosen": -302.2297058105469,
+      "logps/rejected": -213.53564453125,
+      "loss": 0.541,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.21759366989135742,
+      "rewards/margins": 1.1542640924453735,
+      "rewards/rejected": -1.371857762336731,
+      "step": 210
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 205.46982898914413,
+      "learning_rate": 4.7661870503597116e-07,
+      "logits/chosen": -3.0621910095214844,
+      "logits/rejected": -3.102663040161133,
+      "logps/chosen": -170.4927978515625,
+      "logps/rejected": -152.03933715820312,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.28674641251564026,
+      "rewards/margins": 0.7877071499824524,
+      "rewards/rejected": -1.074453592300415,
+      "step": 220
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 302.7432924096287,
+      "learning_rate": 4.730215827338129e-07,
+      "logits/chosen": -2.86230731010437,
+      "logits/rejected": -2.8046908378601074,
+      "logps/chosen": -277.9963684082031,
+      "logps/rejected": -311.2679748535156,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9725619554519653,
+      "rewards/margins": 2.1599535942077637,
+      "rewards/rejected": -3.1325161457061768,
+      "step": 230
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 192.16860695693003,
+      "learning_rate": 4.6942446043165467e-07,
+      "logits/chosen": -2.9580187797546387,
+      "logits/rejected": -2.9902093410491943,
+      "logps/chosen": -244.5732421875,
+      "logps/rejected": -234.186767578125,
+      "loss": 0.5837,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.015125697478652,
+      "rewards/margins": 0.9667981863021851,
+      "rewards/rejected": -0.9819238781929016,
+      "step": 240
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 193.43307883430833,
+      "learning_rate": 4.6582733812949637e-07,
+      "logits/chosen": -3.113363742828369,
+      "logits/rejected": -3.1423449516296387,
+      "logps/chosen": -240.1184539794922,
+      "logps/rejected": -243.276123046875,
+      "loss": 0.5109,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8149322271347046,
+      "rewards/margins": 0.7386936545372009,
+      "rewards/rejected": -1.5536259412765503,
+      "step": 250
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 183.86586127077544,
+      "learning_rate": 4.622302158273381e-07,
+      "logits/chosen": -3.1412100791931152,
+      "logits/rejected": -3.0982837677001953,
+      "logps/chosen": -221.32955932617188,
+      "logps/rejected": -194.5541534423828,
+      "loss": 0.5086,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8559683561325073,
+      "rewards/margins": 0.9293031692504883,
+      "rewards/rejected": -1.785271406173706,
+      "step": 260
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 252.90755166196357,
+      "learning_rate": 4.586330935251798e-07,
+      "logits/chosen": -3.0562283992767334,
+      "logits/rejected": -3.016035556793213,
+      "logps/chosen": -288.05792236328125,
+      "logps/rejected": -246.1804656982422,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.19852256774902344,
+      "rewards/margins": 0.7442554235458374,
+      "rewards/rejected": -0.9427779912948608,
+      "step": 270
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 211.83556122462332,
+      "learning_rate": 4.550359712230216e-07,
+      "logits/chosen": -2.89155650138855,
+      "logits/rejected": -2.8994014263153076,
+      "logps/chosen": -286.43341064453125,
+      "logps/rejected": -301.67144775390625,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.3442370295524597,
+      "rewards/margins": 1.805853247642517,
+      "rewards/rejected": -2.1500906944274902,
+      "step": 280
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 418.7444892216886,
+      "learning_rate": 4.5143884892086333e-07,
+      "logits/chosen": -2.7451279163360596,
+      "logits/rejected": -2.687638759613037,
+      "logps/chosen": -267.42816162109375,
+      "logps/rejected": -184.01612854003906,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1652831733226776,
+      "rewards/margins": 1.0949604511260986,
+      "rewards/rejected": -0.9296773076057434,
+      "step": 290
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 257.51596842504944,
+      "learning_rate": 4.4784172661870503e-07,
+      "logits/chosen": -2.8880581855773926,
+      "logits/rejected": -2.859224796295166,
+      "logps/chosen": -228.3848114013672,
+      "logps/rejected": -243.755859375,
+      "loss": 0.644,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.26425519585609436,
+      "rewards/margins": 0.545816957950592,
+      "rewards/rejected": -0.810072124004364,
+      "step": 300
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 146.22803074809437,
+      "learning_rate": 4.4424460431654673e-07,
+      "logits/chosen": -2.8631234169006348,
+      "logits/rejected": -2.929395914077759,
+      "logps/chosen": -157.17486572265625,
+      "logps/rejected": -203.16934204101562,
+      "loss": 0.5586,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0279247760772705,
+      "rewards/margins": 2.014747142791748,
+      "rewards/rejected": -3.0426723957061768,
+      "step": 310
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 255.32693322709616,
+      "learning_rate": 4.4064748201438843e-07,
+      "logits/chosen": -2.983949661254883,
+      "logits/rejected": -3.011338233947754,
+      "logps/chosen": -186.72018432617188,
+      "logps/rejected": -193.33999633789062,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3878747522830963,
+      "rewards/margins": 1.1526678800582886,
+      "rewards/rejected": -1.5405426025390625,
+      "step": 320
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 238.21102592762895,
+      "learning_rate": 4.370503597122302e-07,
+      "logits/chosen": -2.962764263153076,
+      "logits/rejected": -2.9538509845733643,
+      "logps/chosen": -277.8987121582031,
+      "logps/rejected": -255.3301544189453,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7843912839889526,
+      "rewards/margins": 1.1306078433990479,
+      "rewards/rejected": -1.914999008178711,
+      "step": 330
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 198.9409546988356,
+      "learning_rate": 4.3345323741007194e-07,
+      "logits/chosen": -2.998401403427124,
+      "logits/rejected": -3.0707414150238037,
+      "logps/chosen": -188.1188201904297,
+      "logps/rejected": -207.1416778564453,
+      "loss": 0.5624,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6548808813095093,
+      "rewards/margins": 1.881704330444336,
+      "rewards/rejected": -2.5365853309631348,
+      "step": 340
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 105.06843294784386,
+      "learning_rate": 4.2985611510791364e-07,
+      "logits/chosen": -2.783904552459717,
+      "logits/rejected": -2.7255444526672363,
+      "logps/chosen": -198.8332061767578,
+      "logps/rejected": -173.35971069335938,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7126100659370422,
+      "rewards/margins": 1.1439498662948608,
+      "rewards/rejected": -1.8565599918365479,
+      "step": 350
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 263.3174694783378,
+      "learning_rate": 4.262589928057554e-07,
+      "logits/chosen": -2.9126217365264893,
+      "logits/rejected": -2.9537577629089355,
+      "logps/chosen": -175.30380249023438,
+      "logps/rejected": -158.21139526367188,
+      "loss": 0.4754,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1357567310333252,
+      "rewards/margins": 2.3206634521484375,
+      "rewards/rejected": -3.4564201831817627,
+      "step": 360
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 177.41732636758604,
+      "learning_rate": 4.226618705035971e-07,
+      "logits/chosen": -3.006334066390991,
+      "logits/rejected": -3.057971477508545,
+      "logps/chosen": -319.56353759765625,
+      "logps/rejected": -296.5467529296875,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.418001651763916,
+      "rewards/margins": 1.0575675964355469,
+      "rewards/rejected": -2.475569248199463,
+      "step": 370
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 218.4833326220091,
+      "learning_rate": 4.1906474820143885e-07,
+      "logits/chosen": -2.894559383392334,
+      "logits/rejected": -2.959474802017212,
+      "logps/chosen": -183.92538452148438,
+      "logps/rejected": -209.09469604492188,
+      "loss": 0.5208,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1497817039489746,
+      "rewards/margins": 0.8618785738945007,
+      "rewards/rejected": -2.01166033744812,
+      "step": 380
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 202.84052669573208,
+      "learning_rate": 4.154676258992806e-07,
+      "logits/chosen": -2.7759809494018555,
+      "logits/rejected": -2.7760024070739746,
+      "logps/chosen": -226.66012573242188,
+      "logps/rejected": -175.62423706054688,
+      "loss": 0.5323,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.1147387027740479,
+      "rewards/margins": 1.6622241735458374,
+      "rewards/rejected": -2.776962995529175,
+      "step": 390
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 182.17649058100815,
+      "learning_rate": 4.118705035971223e-07,
+      "logits/chosen": -3.002817153930664,
+      "logits/rejected": -2.912900686264038,
+      "logps/chosen": -181.59104919433594,
+      "logps/rejected": -180.56857299804688,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.058836579322815,
+      "rewards/margins": 1.3610327243804932,
+      "rewards/rejected": -2.4198696613311768,
+      "step": 400
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 161.96603302480173,
+      "learning_rate": 4.08273381294964e-07,
+      "logits/chosen": -2.995290517807007,
+      "logits/rejected": -2.921790361404419,
+      "logps/chosen": -279.0407409667969,
+      "logps/rejected": -241.841796875,
+      "loss": 0.5213,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12093071639537811,
+      "rewards/margins": 1.5623955726623535,
+      "rewards/rejected": -1.6833263635635376,
+      "step": 410
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 201.93847203217072,
+      "learning_rate": 4.046762589928057e-07,
+      "logits/chosen": -2.744863510131836,
+      "logits/rejected": -2.775908946990967,
+      "logps/chosen": -235.76181030273438,
+      "logps/rejected": -256.9134826660156,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.520436704158783,
+      "rewards/margins": 1.4860963821411133,
+      "rewards/rejected": -2.006533145904541,
+      "step": 420
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 394.9065983141447,
+      "learning_rate": 4.0107913669064746e-07,
+      "logits/chosen": -2.8987364768981934,
+      "logits/rejected": -3.0120251178741455,
+      "logps/chosen": -199.3771514892578,
+      "logps/rejected": -276.7981872558594,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3013836443424225,
+      "rewards/margins": 2.102478504180908,
+      "rewards/rejected": -2.4038619995117188,
+      "step": 430
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 154.86934618345205,
+      "learning_rate": 3.974820143884892e-07,
+      "logits/chosen": -2.8435750007629395,
+      "logits/rejected": -2.9437308311462402,
+      "logps/chosen": -166.1263885498047,
+      "logps/rejected": -191.59690856933594,
+      "loss": 0.4899,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5653941631317139,
+      "rewards/margins": 1.7190685272216797,
+      "rewards/rejected": -3.2844626903533936,
+      "step": 440
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 140.28100702480398,
+      "learning_rate": 3.938848920863309e-07,
+      "logits/chosen": -2.9444668292999268,
+      "logits/rejected": -2.960024118423462,
+      "logps/chosen": -230.9547119140625,
+      "logps/rejected": -249.8640899658203,
+      "loss": 0.5214,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.988498330116272,
+      "rewards/margins": 0.8448504209518433,
+      "rewards/rejected": -2.833348512649536,
+      "step": 450
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 251.261088110334,
+      "learning_rate": 3.9028776978417266e-07,
+      "logits/chosen": -2.956639289855957,
+      "logits/rejected": -2.9189281463623047,
+      "logps/chosen": -248.42074584960938,
+      "logps/rejected": -259.29547119140625,
+      "loss": 0.5548,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4849539995193481,
+      "rewards/margins": 1.1761761903762817,
+      "rewards/rejected": -2.66113018989563,
+      "step": 460
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 157.1165843220562,
+      "learning_rate": 3.8669064748201436e-07,
+      "logits/chosen": -2.9550623893737793,
+      "logits/rejected": -2.995311975479126,
+      "logps/chosen": -175.0579071044922,
+      "logps/rejected": -205.593505859375,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.3374639749526978,
+      "rewards/margins": 2.801967144012451,
+      "rewards/rejected": -4.139430999755859,
+      "step": 470
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 239.1056515400595,
+      "learning_rate": 3.830935251798561e-07,
+      "logits/chosen": -2.902780771255493,
+      "logits/rejected": -2.9937572479248047,
+      "logps/chosen": -168.61141967773438,
+      "logps/rejected": -209.18344116210938,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.3175175189971924,
+      "rewards/margins": 1.5122530460357666,
+      "rewards/rejected": -3.82977032661438,
+      "step": 480
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 148.09456662862073,
+      "learning_rate": 3.7949640287769787e-07,
+      "logits/chosen": -2.844252347946167,
+      "logits/rejected": -2.7700600624084473,
+      "logps/chosen": -273.8468933105469,
+      "logps/rejected": -166.2052764892578,
+      "loss": 0.5295,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9907764196395874,
+      "rewards/margins": 0.7116690874099731,
+      "rewards/rejected": -2.7024455070495605,
+      "step": 490
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 212.94009211428397,
+      "learning_rate": 3.7589928057553957e-07,
+      "logits/chosen": -2.781285285949707,
+      "logits/rejected": -2.8481528759002686,
+      "logps/chosen": -287.84381103515625,
+      "logps/rejected": -233.2913818359375,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6348762512207031,
+      "rewards/margins": 2.143733501434326,
+      "rewards/rejected": -3.7786097526550293,
+      "step": 500
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 347.65521206077733,
+      "learning_rate": 3.7230215827338127e-07,
+      "logits/chosen": -2.8734822273254395,
+      "logits/rejected": -2.871880531311035,
+      "logps/chosen": -236.97799682617188,
+      "logps/rejected": -269.2080078125,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1686577796936035,
+      "rewards/margins": 2.460658550262451,
+      "rewards/rejected": -4.629316806793213,
+      "step": 510
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 17.093431174075842,
+      "learning_rate": 3.6870503597122297e-07,
+      "logits/chosen": -2.881934404373169,
+      "logits/rejected": -2.821448802947998,
+      "logps/chosen": -159.77235412597656,
+      "logps/rejected": -186.3824920654297,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8075105547904968,
+      "rewards/margins": 3.561521053314209,
+      "rewards/rejected": -4.36903190612793,
+      "step": 520
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 10.962962944310295,
+      "learning_rate": 3.651079136690647e-07,
+      "logits/chosen": -2.906359910964966,
+      "logits/rejected": -2.8581392765045166,
+      "logps/chosen": -248.25723266601562,
+      "logps/rejected": -273.17218017578125,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.4104425013065338,
+      "rewards/margins": 6.661971092224121,
+      "rewards/rejected": -6.251528739929199,
+      "step": 530
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 19.397740520521022,
+      "learning_rate": 3.615107913669064e-07,
+      "logits/chosen": -2.8592209815979004,
+      "logits/rejected": -2.821002721786499,
+      "logps/chosen": -153.77841186523438,
+      "logps/rejected": -253.50192260742188,
+      "loss": 0.1241,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.922268271446228,
+      "rewards/margins": 4.912806510925293,
+      "rewards/rejected": -5.8350749015808105,
+      "step": 540
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 31.58327548524221,
+      "learning_rate": 3.579136690647482e-07,
+      "logits/chosen": -2.772770881652832,
+      "logits/rejected": -2.7921929359436035,
+      "logps/chosen": -147.0404052734375,
+      "logps/rejected": -281.2487487792969,
+      "loss": 0.1004,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5385946035385132,
+      "rewards/margins": 6.8628716468811035,
+      "rewards/rejected": -6.324276924133301,
+      "step": 550
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 10.777257966533346,
+      "learning_rate": 3.5431654676258993e-07,
+      "logits/chosen": -2.779730796813965,
+      "logits/rejected": -2.801069498062134,
+      "logps/chosen": -153.9626922607422,
+      "logps/rejected": -171.65768432617188,
+      "loss": 0.1374,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.910001516342163,
+      "rewards/margins": 4.283631324768066,
+      "rewards/rejected": -6.193632125854492,
+      "step": 560
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 23.32063703058829,
+      "learning_rate": 3.5071942446043163e-07,
+      "logits/chosen": -2.872498035430908,
+      "logits/rejected": -2.884324789047241,
+      "logps/chosen": -267.8298034667969,
+      "logps/rejected": -381.44049072265625,
+      "loss": 0.106,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.8878796100616455,
+      "rewards/margins": 10.999748229980469,
+      "rewards/rejected": -8.111869812011719,
+      "step": 570
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 31.976606028398866,
+      "learning_rate": 3.471223021582734e-07,
+      "logits/chosen": -2.8648922443389893,
+      "logits/rejected": -2.8326311111450195,
+      "logps/chosen": -198.3783721923828,
+      "logps/rejected": -266.2000427246094,
+      "loss": 0.1014,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.19176311790943146,
+      "rewards/margins": 7.178011894226074,
+      "rewards/rejected": -6.986248970031738,
+      "step": 580
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 15.736933231868846,
+      "learning_rate": 3.435251798561151e-07,
+      "logits/chosen": -2.8333282470703125,
+      "logits/rejected": -2.9636716842651367,
+      "logps/chosen": -206.75283813476562,
+      "logps/rejected": -282.7793273925781,
+      "loss": 0.0985,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.6388500332832336,
+      "rewards/margins": 4.777499198913574,
+      "rewards/rejected": -5.416348934173584,
+      "step": 590
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 17.063403322364767,
+      "learning_rate": 3.3992805755395684e-07,
+      "logits/chosen": -2.919651508331299,
+      "logits/rejected": -2.996105670928955,
+      "logps/chosen": -192.1748504638672,
+      "logps/rejected": -310.4754333496094,
+      "loss": 0.1476,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5340520143508911,
+      "rewards/margins": 7.349068641662598,
+      "rewards/rejected": -6.815016269683838,
+      "step": 600
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 20.21744560573151,
+      "learning_rate": 3.3633093525179854e-07,
+      "logits/chosen": -2.9296090602874756,
+      "logits/rejected": -2.7481861114501953,
+      "logps/chosen": -203.38052368164062,
+      "logps/rejected": -244.37014770507812,
+      "loss": 0.1342,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.0323004722595215,
+      "rewards/margins": 8.135452270507812,
+      "rewards/rejected": -7.103150844573975,
+      "step": 610
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 16.433861008142927,
+      "learning_rate": 3.3273381294964024e-07,
+      "logits/chosen": -2.9079694747924805,
+      "logits/rejected": -2.8130078315734863,
+      "logps/chosen": -186.45281982421875,
+      "logps/rejected": -252.8771514892578,
+      "loss": 0.0962,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.2089195251464844,
+      "rewards/margins": 5.797079563140869,
+      "rewards/rejected": -4.588160037994385,
+      "step": 620
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 29.799329342774556,
+      "learning_rate": 3.29136690647482e-07,
+      "logits/chosen": -2.971191883087158,
+      "logits/rejected": -2.8457350730895996,
+      "logps/chosen": -229.301513671875,
+      "logps/rejected": -390.90899658203125,
+      "loss": 0.1191,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 3.2872531414031982,
+      "rewards/margins": 14.088516235351562,
+      "rewards/rejected": -10.801262855529785,
+      "step": 630
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 13.229591095093818,
+      "learning_rate": 3.255395683453237e-07,
+      "logits/chosen": -2.7274880409240723,
+      "logits/rejected": -2.754389524459839,
+      "logps/chosen": -139.60235595703125,
+      "logps/rejected": -234.2040557861328,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.6343626976013184,
+      "rewards/margins": 7.628392696380615,
+      "rewards/rejected": -6.994029998779297,
+      "step": 640
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 8.348959533199398,
+      "learning_rate": 3.2194244604316545e-07,
+      "logits/chosen": -2.8291614055633545,
+      "logits/rejected": -2.768712282180786,
+      "logps/chosen": -156.22329711914062,
+      "logps/rejected": -280.3912658691406,
+      "loss": 0.1206,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6103225946426392,
+      "rewards/margins": 5.600022315979004,
+      "rewards/rejected": -6.2103447914123535,
+      "step": 650
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 6.080339854051856,
+      "learning_rate": 3.183453237410072e-07,
+      "logits/chosen": -2.8922338485717773,
+      "logits/rejected": -3.0048136711120605,
+      "logps/chosen": -185.24578857421875,
+      "logps/rejected": -273.0670166015625,
+      "loss": 0.1864,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.4969745874404907,
+      "rewards/margins": 6.30242395401001,
+      "rewards/rejected": -7.799398899078369,
+      "step": 660
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 8.711710883557748,
+      "learning_rate": 3.147482014388489e-07,
+      "logits/chosen": -2.827256679534912,
+      "logits/rejected": -2.9615511894226074,
+      "logps/chosen": -190.73292541503906,
+      "logps/rejected": -271.886962890625,
+      "loss": 0.1163,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.6087120771408081,
+      "rewards/margins": 7.128113746643066,
+      "rewards/rejected": -7.7368268966674805,
+      "step": 670
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 23.330004354397758,
+      "learning_rate": 3.1115107913669066e-07,
+      "logits/chosen": -3.0536856651306152,
+      "logits/rejected": -3.0247254371643066,
+      "logps/chosen": -250.87240600585938,
+      "logps/rejected": -255.5919952392578,
+      "loss": 0.1319,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.45528966188430786,
+      "rewards/margins": 6.966174125671387,
+      "rewards/rejected": -6.510884761810303,
+      "step": 680
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 23.99609139468775,
+      "learning_rate": 3.0755395683453236e-07,
+      "logits/chosen": -2.9475927352905273,
+      "logits/rejected": -3.0371546745300293,
+      "logps/chosen": -187.01266479492188,
+      "logps/rejected": -247.2963409423828,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.9167653918266296,
+      "rewards/margins": 7.090806007385254,
+      "rewards/rejected": -8.007570266723633,
+      "step": 690
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 28.810275406507504,
+      "learning_rate": 3.039568345323741e-07,
+      "logits/chosen": -2.9736738204956055,
+      "logits/rejected": -3.023808240890503,
+      "logps/chosen": -155.4862060546875,
+      "logps/rejected": -202.7901153564453,
+      "loss": 0.1,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.3716559410095215,
+      "rewards/margins": 6.325069427490234,
+      "rewards/rejected": -8.696724891662598,
+      "step": 700
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 28.433296481008185,
+      "learning_rate": 3.003597122302158e-07,
+      "logits/chosen": -2.9116153717041016,
+      "logits/rejected": -2.9804043769836426,
+      "logps/chosen": -283.181396484375,
+      "logps/rejected": -294.97491455078125,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.19354362785816193,
+      "rewards/margins": 8.582574844360352,
+      "rewards/rejected": -8.389032363891602,
+      "step": 710
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 8.876311353316893,
+      "learning_rate": 2.967625899280575e-07,
+      "logits/chosen": -2.9362125396728516,
+      "logits/rejected": -3.0337347984313965,
+      "logps/chosen": -181.9010009765625,
+      "logps/rejected": -255.519775390625,
+      "loss": 0.1098,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.3271546959877014,
+      "rewards/margins": 7.4435715675354,
+      "rewards/rejected": -7.770726203918457,
+      "step": 720
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 9.661909624634754,
+      "learning_rate": 2.9316546762589927e-07,
+      "logits/chosen": -3.0636532306671143,
+      "logits/rejected": -3.1014134883880615,
+      "logps/chosen": -215.9973602294922,
+      "logps/rejected": -313.1678161621094,
+      "loss": 0.0814,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.42110705375671387,
+      "rewards/margins": 10.026618957519531,
+      "rewards/rejected": -10.447726249694824,
+      "step": 730
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 16.74604259022382,
+      "learning_rate": 2.8956834532374097e-07,
+      "logits/chosen": -3.016714334487915,
+      "logits/rejected": -2.938993453979492,
+      "logps/chosen": -242.8096160888672,
+      "logps/rejected": -330.51348876953125,
+      "loss": 0.0879,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.959947407245636,
+      "rewards/margins": 7.937321662902832,
+      "rewards/rejected": -8.897270202636719,
+      "step": 740
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 21.06683152485559,
+      "learning_rate": 2.859712230215827e-07,
+      "logits/chosen": -2.8868894577026367,
+      "logits/rejected": -2.8016741275787354,
+      "logps/chosen": -141.5897674560547,
+      "logps/rejected": -192.32943725585938,
+      "loss": 0.0998,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.15523338317871094,
+      "rewards/margins": 7.094580173492432,
+      "rewards/rejected": -7.249814033508301,
+      "step": 750
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 31.082677373086035,
+      "learning_rate": 2.8237410071942447e-07,
+      "logits/chosen": -3.1299731731414795,
+      "logits/rejected": -3.0648064613342285,
+      "logps/chosen": -195.8338165283203,
+      "logps/rejected": -223.024169921875,
+      "loss": 0.1339,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.07549004256725311,
+      "rewards/margins": 8.427389144897461,
+      "rewards/rejected": -8.502878189086914,
+      "step": 760
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 43.39594697344513,
+      "learning_rate": 2.7877697841726617e-07,
+      "logits/chosen": -2.9652228355407715,
+      "logits/rejected": -3.046382427215576,
+      "logps/chosen": -220.59426879882812,
+      "logps/rejected": -365.4892578125,
+      "loss": 0.0799,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.5180127620697021,
+      "rewards/margins": 8.326019287109375,
+      "rewards/rejected": -7.80800724029541,
+      "step": 770
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 19.539022726952826,
+      "learning_rate": 2.751798561151079e-07,
+      "logits/chosen": -3.0099740028381348,
+      "logits/rejected": -3.0176591873168945,
+      "logps/chosen": -228.35977172851562,
+      "logps/rejected": -307.06292724609375,
+      "loss": 0.0939,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.7477496862411499,
+      "rewards/margins": 6.7176995277404785,
+      "rewards/rejected": -7.465450286865234,
+      "step": 780
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 28.726157406769754,
+      "learning_rate": 2.7158273381294963e-07,
+      "logits/chosen": -3.113996982574463,
+      "logits/rejected": -3.1443333625793457,
+      "logps/chosen": -273.08770751953125,
+      "logps/rejected": -364.9424133300781,
+      "loss": 0.203,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.5305331945419312,
+      "rewards/margins": 10.640276908874512,
+      "rewards/rejected": -9.10974407196045,
+      "step": 790
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 47.13980834678601,
+      "learning_rate": 2.679856115107914e-07,
+      "logits/chosen": -3.059725761413574,
+      "logits/rejected": -3.009997844696045,
+      "logps/chosen": -298.9270935058594,
+      "logps/rejected": -321.82427978515625,
+      "loss": 0.0951,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.40496888756752014,
+      "rewards/margins": 8.552021026611328,
+      "rewards/rejected": -8.147050857543945,
+      "step": 800
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 40.67896985913433,
+      "learning_rate": 2.643884892086331e-07,
+      "logits/chosen": -2.9675583839416504,
+      "logits/rejected": -2.9391565322875977,
+      "logps/chosen": -193.52569580078125,
+      "logps/rejected": -270.5869140625,
+      "loss": 0.1291,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.095348596572876,
+      "rewards/margins": 9.404745101928711,
+      "rewards/rejected": -8.309396743774414,
+      "step": 810
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 0.7304638838059418,
+      "learning_rate": 2.607913669064748e-07,
+      "logits/chosen": -3.051098346710205,
+      "logits/rejected": -3.031184673309326,
+      "logps/chosen": -276.66949462890625,
+      "logps/rejected": -325.5706481933594,
+      "loss": 0.2483,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 2.204752206802368,
+      "rewards/margins": 8.472108840942383,
+      "rewards/rejected": -6.26735782623291,
+      "step": 820
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 51.38852311030539,
+      "learning_rate": 2.5719424460431653e-07,
+      "logits/chosen": -2.971363067626953,
+      "logits/rejected": -2.7650504112243652,
+      "logps/chosen": -227.90182495117188,
+      "logps/rejected": -277.134765625,
+      "loss": 0.0864,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.8440443277359009,
+      "rewards/margins": 8.721940994262695,
+      "rewards/rejected": -6.877896308898926,
+      "step": 830
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 27.97321903508496,
+      "learning_rate": 2.5359712230215824e-07,
+      "logits/chosen": -2.748213052749634,
+      "logits/rejected": -2.9322519302368164,
+      "logps/chosen": -214.3332977294922,
+      "logps/rejected": -324.88092041015625,
+      "loss": 0.1023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5567120313644409,
+      "rewards/margins": 10.445902824401855,
+      "rewards/rejected": -9.889190673828125,
+      "step": 840
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 25.914111546778514,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.8877971172332764,
+      "logits/rejected": -2.816826581954956,
+      "logps/chosen": -204.908203125,
+      "logps/rejected": -430.1329650878906,
+      "loss": 0.1047,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.22576239705085754,
+      "rewards/margins": 20.19011878967285,
+      "rewards/rejected": -19.96435546875,
+      "step": 850
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 1.1980659785434056,
+      "learning_rate": 2.464028776978417e-07,
+      "logits/chosen": -2.936771869659424,
+      "logits/rejected": -2.96122407913208,
+      "logps/chosen": -255.11215209960938,
+      "logps/rejected": -308.09161376953125,
+      "loss": 0.0862,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.9314340353012085,
+      "rewards/margins": 9.529711723327637,
+      "rewards/rejected": -8.598278045654297,
+      "step": 860
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 27.693072917563903,
+      "learning_rate": 2.4280575539568344e-07,
+      "logits/chosen": -2.6308648586273193,
+      "logits/rejected": -2.745168447494507,
+      "logps/chosen": -194.28121948242188,
+      "logps/rejected": -305.8986511230469,
+      "loss": 0.0968,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.185044288635254,
+      "rewards/margins": 8.040155410766602,
+      "rewards/rejected": -10.225198745727539,
+      "step": 870
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 40.41147122052846,
+      "learning_rate": 2.392086330935252e-07,
+      "logits/chosen": -2.8742213249206543,
+      "logits/rejected": -2.8451921939849854,
+      "logps/chosen": -241.3814697265625,
+      "logps/rejected": -313.4134521484375,
+      "loss": 0.2677,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.6791108846664429,
+      "rewards/margins": 9.191535949707031,
+      "rewards/rejected": -7.512425899505615,
+      "step": 880
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 71.6126200211681,
+      "learning_rate": 2.356115107913669e-07,
+      "logits/chosen": -2.9680380821228027,
+      "logits/rejected": -2.8588905334472656,
+      "logps/chosen": -178.02896118164062,
+      "logps/rejected": -293.750244140625,
+      "loss": 0.0765,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.8559761047363281,
+      "rewards/margins": 9.371949195861816,
+      "rewards/rejected": -10.227925300598145,
+      "step": 890
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 61.57298874023214,
+      "learning_rate": 2.3201438848920862e-07,
+      "logits/chosen": -2.919063091278076,
+      "logits/rejected": -2.9276976585388184,
+      "logps/chosen": -173.81768798828125,
+      "logps/rejected": -284.75054931640625,
+      "loss": 0.0982,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.0116955041885376,
+      "rewards/margins": 8.733976364135742,
+      "rewards/rejected": -7.722282409667969,
+      "step": 900
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 29.6755083957111,
+      "learning_rate": 2.2841726618705035e-07,
+      "logits/chosen": -2.9070231914520264,
+      "logits/rejected": -2.838956356048584,
+      "logps/chosen": -219.24081420898438,
+      "logps/rejected": -275.2736511230469,
+      "loss": 0.1104,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.1092593669891357,
+      "rewards/margins": 6.717539310455322,
+      "rewards/rejected": -7.826798915863037,
+      "step": 910
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 88.06965761995421,
+      "learning_rate": 2.2482014388489208e-07,
+      "logits/chosen": -2.7650623321533203,
+      "logits/rejected": -2.985419750213623,
+      "logps/chosen": -209.66958618164062,
+      "logps/rejected": -307.84808349609375,
+      "loss": 0.1387,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.090811848640442,
+      "rewards/margins": 8.861490249633789,
+      "rewards/rejected": -7.770678520202637,
+      "step": 920
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 14.973419395885822,
+      "learning_rate": 2.212230215827338e-07,
+      "logits/chosen": -2.8417439460754395,
+      "logits/rejected": -2.875882863998413,
+      "logps/chosen": -176.11224365234375,
+      "logps/rejected": -296.80194091796875,
+      "loss": 0.1462,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.6913576126098633,
+      "rewards/margins": 9.900197982788086,
+      "rewards/rejected": -9.208839416503906,
+      "step": 930
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 19.94743902740955,
+      "learning_rate": 2.1762589928057553e-07,
+      "logits/chosen": -2.825326442718506,
+      "logits/rejected": -2.7393977642059326,
+      "logps/chosen": -183.79501342773438,
+      "logps/rejected": -237.4878387451172,
+      "loss": 0.0987,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1172792911529541,
+      "rewards/margins": 5.7345991134643555,
+      "rewards/rejected": -5.851877689361572,
+      "step": 940
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 24.84797812605884,
+      "learning_rate": 2.1402877697841726e-07,
+      "logits/chosen": -2.966536283493042,
+      "logits/rejected": -2.9578917026519775,
+      "logps/chosen": -203.7595672607422,
+      "logps/rejected": -262.04974365234375,
+      "loss": 0.1062,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.500684380531311,
+      "rewards/margins": 6.407332420349121,
+      "rewards/rejected": -5.906649112701416,
+      "step": 950
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 10.723678972332182,
+      "learning_rate": 2.1043165467625899e-07,
+      "logits/chosen": -3.024386405944824,
+      "logits/rejected": -2.9682140350341797,
+      "logps/chosen": -290.12860107421875,
+      "logps/rejected": -395.7617492675781,
+      "loss": 0.0888,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.9799671173095703,
+      "rewards/margins": 10.694775581359863,
+      "rewards/rejected": -8.71480655670166,
+      "step": 960
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 107.78338392646336,
+      "learning_rate": 2.068345323741007e-07,
+      "logits/chosen": -2.7938714027404785,
+      "logits/rejected": -2.8441925048828125,
+      "logps/chosen": -113.68382263183594,
+      "logps/rejected": -215.2233428955078,
+      "loss": 0.1476,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5707006454467773,
+      "rewards/margins": 6.20760440826416,
+      "rewards/rejected": -7.7783050537109375,
+      "step": 970
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 12.51020681306747,
+      "learning_rate": 2.0323741007194244e-07,
+      "logits/chosen": -2.961688995361328,
+      "logits/rejected": -2.7563016414642334,
+      "logps/chosen": -224.689453125,
+      "logps/rejected": -250.39193725585938,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.786786675453186,
+      "rewards/margins": 5.6869611740112305,
+      "rewards/rejected": -7.473748207092285,
+      "step": 980
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 22.530969988611286,
+      "learning_rate": 1.9964028776978417e-07,
+      "logits/chosen": -2.937544345855713,
+      "logits/rejected": -2.9909420013427734,
+      "logps/chosen": -186.51730346679688,
+      "logps/rejected": -228.6781463623047,
+      "loss": 0.09,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.5189081430435181,
+      "rewards/margins": 6.764664649963379,
+      "rewards/rejected": -7.283572196960449,
+      "step": 990
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 13.752486135555781,
+      "learning_rate": 1.960431654676259e-07,
+      "logits/chosen": -3.005913019180298,
+      "logits/rejected": -2.9097745418548584,
+      "logps/chosen": -204.80996704101562,
+      "logps/rejected": -216.8582763671875,
+      "loss": 0.0884,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.1967257261276245,
+      "rewards/margins": 7.3372955322265625,
+      "rewards/rejected": -8.534021377563477,
+      "step": 1000
+    },
+    {
+      "epoch": 1.94,
+      "eval_logits/chosen": -2.976330518722534,
+      "eval_logits/rejected": -2.9599409103393555,
+      "eval_logps/chosen": -225.5250244140625,
+      "eval_logps/rejected": -230.1306610107422,
+      "eval_loss": 0.5467228293418884,
+      "eval_rewards/accuracies": 0.7024999856948853,
+      "eval_rewards/chosen": -3.4828414916992188,
+      "eval_rewards/margins": 2.553647041320801,
+      "eval_rewards/rejected": -6.036489486694336,
+      "eval_runtime": 169.6259,
+      "eval_samples_per_second": 18.606,
+      "eval_steps_per_second": 0.295,
+      "step": 1000
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 84.31680232274636,
+      "learning_rate": 1.9244604316546762e-07,
+      "logits/chosen": -3.030494451522827,
+      "logits/rejected": -3.0824122428894043,
+      "logps/chosen": -208.97262573242188,
+      "logps/rejected": -380.12371826171875,
+      "loss": 0.131,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.10666930675506592,
+      "rewards/margins": 10.088615417480469,
+      "rewards/rejected": -9.98194694519043,
+      "step": 1010
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 16.33806610185454,
+      "learning_rate": 1.8884892086330935e-07,
+      "logits/chosen": -2.9731550216674805,
+      "logits/rejected": -2.98013973236084,
+      "logps/chosen": -201.14505004882812,
+      "logps/rejected": -301.49871826171875,
+      "loss": 0.0921,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.5621670484542847,
+      "rewards/margins": 10.991891860961914,
+      "rewards/rejected": -10.429725646972656,
+      "step": 1020
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 5.953845816794541,
+      "learning_rate": 1.8525179856115105e-07,
+      "logits/chosen": -2.810067653656006,
+      "logits/rejected": -2.896484851837158,
+      "logps/chosen": -170.7087860107422,
+      "logps/rejected": -305.812744140625,
+      "loss": 0.2128,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.03446235507726669,
+      "rewards/margins": 8.075230598449707,
+      "rewards/rejected": -8.109692573547363,
+      "step": 1030
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 25.008245531662734,
+      "learning_rate": 1.816546762589928e-07,
+      "logits/chosen": -2.9846489429473877,
+      "logits/rejected": -2.7843635082244873,
+      "logps/chosen": -199.68411254882812,
+      "logps/rejected": -286.22967529296875,
+      "loss": 0.0662,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.3490269184112549,
+      "rewards/margins": 9.136482238769531,
+      "rewards/rejected": -7.787455081939697,
+      "step": 1040
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 5.633840150274412,
+      "learning_rate": 1.7805755395683453e-07,
+      "logits/chosen": -2.7847142219543457,
+      "logits/rejected": -2.885359287261963,
+      "logps/chosen": -219.5730438232422,
+      "logps/rejected": -254.0897674560547,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.2533214092254639,
+      "rewards/margins": 8.608622550964355,
+      "rewards/rejected": -9.861943244934082,
+      "step": 1050
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 7.078559278301402,
+      "learning_rate": 1.7446043165467626e-07,
+      "logits/chosen": -2.852372407913208,
+      "logits/rejected": -2.7808871269226074,
+      "logps/chosen": -215.5196075439453,
+      "logps/rejected": -262.0018005371094,
+      "loss": 0.0832,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6660928130149841,
+      "rewards/margins": 8.312829971313477,
+      "rewards/rejected": -8.978921890258789,
+      "step": 1060
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 59.08802888748966,
+      "learning_rate": 1.7086330935251798e-07,
+      "logits/chosen": -2.7517917156219482,
+      "logits/rejected": -2.806405544281006,
+      "logps/chosen": -172.39793395996094,
+      "logps/rejected": -225.39071655273438,
+      "loss": 0.0657,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.16179580986499786,
+      "rewards/margins": 9.866128921508789,
+      "rewards/rejected": -10.027925491333008,
+      "step": 1070
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 17.218454632279617,
+      "learning_rate": 1.6726618705035968e-07,
+      "logits/chosen": -2.901048183441162,
+      "logits/rejected": -2.8675291538238525,
+      "logps/chosen": -256.65814208984375,
+      "logps/rejected": -278.4682312011719,
+      "loss": 0.0809,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.1122452020645142,
+      "rewards/margins": 8.697083473205566,
+      "rewards/rejected": -9.80932903289795,
+      "step": 1080
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 5.5094848478152585,
+      "learning_rate": 1.6366906474820144e-07,
+      "logits/chosen": -2.7242445945739746,
+      "logits/rejected": -2.590859889984131,
+      "logps/chosen": -196.37344360351562,
+      "logps/rejected": -369.8520812988281,
+      "loss": 0.0932,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.2810817956924438,
+      "rewards/margins": 14.394078254699707,
+      "rewards/rejected": -15.67516040802002,
+      "step": 1090
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 4.35723362011521,
+      "learning_rate": 1.6007194244604316e-07,
+      "logits/chosen": -2.9155492782592773,
+      "logits/rejected": -2.9282360076904297,
+      "logps/chosen": -253.76278686523438,
+      "logps/rejected": -349.87811279296875,
+      "loss": 0.0685,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.497481107711792,
+      "rewards/margins": 10.071554183959961,
+      "rewards/rejected": -9.574074745178223,
+      "step": 1100
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 27.13694624173697,
+      "learning_rate": 1.564748201438849e-07,
+      "logits/chosen": -2.9795854091644287,
+      "logits/rejected": -2.912909746170044,
+      "logps/chosen": -204.10121154785156,
+      "logps/rejected": -311.55316162109375,
+      "loss": 0.0655,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.9577583074569702,
+      "rewards/margins": 11.63161563873291,
+      "rewards/rejected": -9.673857688903809,
+      "step": 1110
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 9.412304352633278,
+      "learning_rate": 1.5287769784172662e-07,
+      "logits/chosen": -2.699101686477661,
+      "logits/rejected": -2.817841053009033,
+      "logps/chosen": -154.8762664794922,
+      "logps/rejected": -330.6363525390625,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.620474100112915,
+      "rewards/margins": 9.67332649230957,
+      "rewards/rejected": -11.29379940032959,
+      "step": 1120
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 3.6214126849238792,
+      "learning_rate": 1.4928057553956832e-07,
+      "logits/chosen": -2.9571373462677,
+      "logits/rejected": -2.856677532196045,
+      "logps/chosen": -171.8690643310547,
+      "logps/rejected": -235.8432159423828,
+      "loss": 0.0925,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -6.249312877655029,
+      "rewards/margins": 6.034473419189453,
+      "rewards/rejected": -12.283784866333008,
+      "step": 1130
+    },
+    {
+      "epoch": 2.21,
+      "grad_norm": 34.86087199990997,
+      "learning_rate": 1.4568345323741007e-07,
+      "logits/chosen": -2.827342987060547,
+      "logits/rejected": -2.9063026905059814,
+      "logps/chosen": -211.0481719970703,
+      "logps/rejected": -349.0590515136719,
+      "loss": 0.0662,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -2.3218135833740234,
+      "rewards/margins": 9.982672691345215,
+      "rewards/rejected": -12.304486274719238,
+      "step": 1140
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 4.714978056289899,
+      "learning_rate": 1.420863309352518e-07,
+      "logits/chosen": -2.8489651679992676,
+      "logits/rejected": -2.886852741241455,
+      "logps/chosen": -204.08822631835938,
+      "logps/rejected": -322.1864929199219,
+      "loss": 0.0854,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09814932197332382,
+      "rewards/margins": 9.744218826293945,
+      "rewards/rejected": -9.842368125915527,
+      "step": 1150
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 2.9947273748809358,
+      "learning_rate": 1.3848920863309352e-07,
+      "logits/chosen": -2.791313886642456,
+      "logits/rejected": -2.819204807281494,
+      "logps/chosen": -269.57891845703125,
+      "logps/rejected": -348.280517578125,
+      "loss": 0.095,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.2503886222839355,
+      "rewards/margins": 8.809748649597168,
+      "rewards/rejected": -11.060136795043945,
+      "step": 1160
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 3.0283319227532206,
+      "learning_rate": 1.3489208633093525e-07,
+      "logits/chosen": -2.9087767601013184,
+      "logits/rejected": -2.84749174118042,
+      "logps/chosen": -266.15948486328125,
+      "logps/rejected": -327.80059814453125,
+      "loss": 0.0629,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.6806023120880127,
+      "rewards/margins": 9.073920249938965,
+      "rewards/rejected": -10.754522323608398,
+      "step": 1170
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 23.204528474486622,
+      "learning_rate": 1.3129496402877695e-07,
+      "logits/chosen": -2.733978748321533,
+      "logits/rejected": -2.7428078651428223,
+      "logps/chosen": -224.3394317626953,
+      "logps/rejected": -264.8204345703125,
+      "loss": 0.0768,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.331441879272461,
+      "rewards/margins": 8.235403060913086,
+      "rewards/rejected": -10.56684398651123,
+      "step": 1180
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 2.386208084053275,
+      "learning_rate": 1.276978417266187e-07,
+      "logits/chosen": -2.870147943496704,
+      "logits/rejected": -2.857393980026245,
+      "logps/chosen": -246.6834716796875,
+      "logps/rejected": -351.1631774902344,
+      "loss": 0.0737,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.3186081647872925,
+      "rewards/margins": 11.430757522583008,
+      "rewards/rejected": -12.749364852905273,
+      "step": 1190
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 5.886073369587131,
+      "learning_rate": 1.2410071942446043e-07,
+      "logits/chosen": -2.635704517364502,
+      "logits/rejected": -2.534574270248413,
+      "logps/chosen": -153.0242462158203,
+      "logps/rejected": -228.0364990234375,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.8014551997184753,
+      "rewards/margins": 9.71955680847168,
+      "rewards/rejected": -10.521011352539062,
+      "step": 1200
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 1.8895213300439382,
+      "learning_rate": 1.2050359712230216e-07,
+      "logits/chosen": -2.8585143089294434,
+      "logits/rejected": -2.8710408210754395,
+      "logps/chosen": -193.92520141601562,
+      "logps/rejected": -308.4482727050781,
+      "loss": 0.0702,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.551455020904541,
+      "rewards/margins": 10.751824378967285,
+      "rewards/rejected": -12.303278923034668,
+      "step": 1210
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 6.1150716672083645,
+      "learning_rate": 1.1690647482014387e-07,
+      "logits/chosen": -2.977564811706543,
+      "logits/rejected": -2.8283185958862305,
+      "logps/chosen": -313.2995300292969,
+      "logps/rejected": -312.1686096191406,
+      "loss": 0.0969,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.5176019668579102,
+      "rewards/margins": 10.63847541809082,
+      "rewards/rejected": -12.156076431274414,
+      "step": 1220
+    },
+    {
+      "epoch": 2.39,
+      "grad_norm": 9.938851512731329,
+      "learning_rate": 1.1330935251798561e-07,
+      "logits/chosen": -2.9206390380859375,
+      "logits/rejected": -2.8288142681121826,
+      "logps/chosen": -225.1664276123047,
+      "logps/rejected": -310.8753662109375,
+      "loss": 0.0297,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9396001696586609,
+      "rewards/margins": 11.567548751831055,
+      "rewards/rejected": -10.627946853637695,
+      "step": 1230
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 2.069636358363744,
+      "learning_rate": 1.0971223021582733e-07,
+      "logits/chosen": -2.859739065170288,
+      "logits/rejected": -2.7655975818634033,
+      "logps/chosen": -255.2501678466797,
+      "logps/rejected": -281.08489990234375,
+      "loss": 0.0568,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.5483479499816895,
+      "rewards/margins": 9.106160163879395,
+      "rewards/rejected": -11.654507637023926,
+      "step": 1240
+    },
+    {
+      "epoch": 2.43,
+      "grad_norm": 10.342520372077749,
+      "learning_rate": 1.0611510791366907e-07,
+      "logits/chosen": -3.0828137397766113,
+      "logits/rejected": -2.9020261764526367,
+      "logps/chosen": -222.02584838867188,
+      "logps/rejected": -289.7030334472656,
+      "loss": 0.0554,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2204093188047409,
+      "rewards/margins": 11.187307357788086,
+      "rewards/rejected": -10.966898918151855,
+      "step": 1250
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 1.6091661711871676,
+      "learning_rate": 1.0251798561151078e-07,
+      "logits/chosen": -2.819439649581909,
+      "logits/rejected": -2.7268171310424805,
+      "logps/chosen": -206.8452606201172,
+      "logps/rejected": -289.83172607421875,
+      "loss": 0.0735,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -5.2341508865356445,
+      "rewards/margins": 8.223525047302246,
+      "rewards/rejected": -13.457676887512207,
+      "step": 1260
+    },
+    {
+      "epoch": 2.47,
+      "grad_norm": 2.9406146592333395,
+      "learning_rate": 9.892086330935251e-08,
+      "logits/chosen": -2.7542366981506348,
+      "logits/rejected": -2.938502073287964,
+      "logps/chosen": -203.71774291992188,
+      "logps/rejected": -298.8545227050781,
+      "loss": 0.0786,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -4.396937370300293,
+      "rewards/margins": 8.35850715637207,
+      "rewards/rejected": -12.755444526672363,
+      "step": 1270
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 5.476946083131152,
+      "learning_rate": 9.532374100719425e-08,
+      "logits/chosen": -2.734649658203125,
+      "logits/rejected": -2.8041725158691406,
+      "logps/chosen": -243.1942596435547,
+      "logps/rejected": -244.3195037841797,
+      "loss": 0.0801,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2553873658180237,
+      "rewards/margins": 9.097474098205566,
+      "rewards/rejected": -8.842086791992188,
+      "step": 1280
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 3.926063875507809,
+      "learning_rate": 9.172661870503596e-08,
+      "logits/chosen": -2.8663649559020996,
+      "logits/rejected": -2.8047869205474854,
+      "logps/chosen": -220.23318481445312,
+      "logps/rejected": -269.9891052246094,
+      "loss": 0.0798,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.644606828689575,
+      "rewards/margins": 8.667104721069336,
+      "rewards/rejected": -11.311712265014648,
+      "step": 1290
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 28.267660173886174,
+      "learning_rate": 8.812949640287769e-08,
+      "logits/chosen": -2.8638269901275635,
+      "logits/rejected": -2.9266953468322754,
+      "logps/chosen": -217.5587615966797,
+      "logps/rejected": -212.8009033203125,
+      "loss": 0.1024,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.6887857913970947,
+      "rewards/margins": 6.493393898010254,
+      "rewards/rejected": -9.18217945098877,
+      "step": 1300
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 12.557325080488198,
+      "learning_rate": 8.453237410071942e-08,
+      "logits/chosen": -2.8576910495758057,
+      "logits/rejected": -2.9612555503845215,
+      "logps/chosen": -258.0310363769531,
+      "logps/rejected": -358.3188171386719,
+      "loss": 0.0534,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.497694730758667,
+      "rewards/margins": 11.422506332397461,
+      "rewards/rejected": -13.920202255249023,
+      "step": 1310
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 2.097907737579444,
+      "learning_rate": 8.093525179856114e-08,
+      "logits/chosen": -2.9389355182647705,
+      "logits/rejected": -2.9012742042541504,
+      "logps/chosen": -263.598876953125,
+      "logps/rejected": -348.38128662109375,
+      "loss": 0.0666,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.6279190182685852,
+      "rewards/margins": 10.751107215881348,
+      "rewards/rejected": -10.123188972473145,
+      "step": 1320
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 3.2585508984060305,
+      "learning_rate": 7.733812949640288e-08,
+      "logits/chosen": -2.897578716278076,
+      "logits/rejected": -2.814047336578369,
+      "logps/chosen": -205.3290252685547,
+      "logps/rejected": -290.0876159667969,
+      "loss": 0.0725,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -5.120051383972168,
+      "rewards/margins": 8.90400505065918,
+      "rewards/rejected": -14.024057388305664,
+      "step": 1330
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 28.5204767402595,
+      "learning_rate": 7.37410071942446e-08,
+      "logits/chosen": -2.6974236965179443,
+      "logits/rejected": -2.763514280319214,
+      "logps/chosen": -230.7462158203125,
+      "logps/rejected": -315.193603515625,
+      "loss": 0.0837,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.1283421516418457,
+      "rewards/margins": 8.708791732788086,
+      "rewards/rejected": -10.837133407592773,
+      "step": 1340
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 13.988032625962447,
+      "learning_rate": 7.014388489208632e-08,
+      "logits/chosen": -2.9279181957244873,
+      "logits/rejected": -2.898128032684326,
+      "logps/chosen": -178.9357147216797,
+      "logps/rejected": -326.0805969238281,
+      "loss": 0.0547,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4182695746421814,
+      "rewards/margins": 9.742707252502441,
+      "rewards/rejected": -9.324437141418457,
+      "step": 1350
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 1.9784130840431484,
+      "learning_rate": 6.654676258992805e-08,
+      "logits/chosen": -2.5293657779693604,
+      "logits/rejected": -2.56721830368042,
+      "logps/chosen": -215.4763946533203,
+      "logps/rejected": -347.84918212890625,
+      "loss": 0.0672,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.11363963782787323,
+      "rewards/margins": 13.453043937683105,
+      "rewards/rejected": -13.566683769226074,
+      "step": 1360
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 9.356111730892692,
+      "learning_rate": 6.294964028776978e-08,
+      "logits/chosen": -2.9187986850738525,
+      "logits/rejected": -2.9558041095733643,
+      "logps/chosen": -185.33265686035156,
+      "logps/rejected": -348.08343505859375,
+      "loss": 0.0603,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.900217056274414,
+      "rewards/margins": 11.395307540893555,
+      "rewards/rejected": -13.295524597167969,
+      "step": 1370
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 1.7310543650629324,
+      "learning_rate": 5.9352517985611505e-08,
+      "logits/chosen": -2.9439430236816406,
+      "logits/rejected": -2.846768856048584,
+      "logps/chosen": -295.23602294921875,
+      "logps/rejected": -313.1105651855469,
+      "loss": 0.0848,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.23460149765014648,
+      "rewards/margins": 8.801981925964355,
+      "rewards/rejected": -8.56737995147705,
+      "step": 1380
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 5.698972392066655,
+      "learning_rate": 5.575539568345323e-08,
+      "logits/chosen": -2.932072162628174,
+      "logits/rejected": -2.9518001079559326,
+      "logps/chosen": -236.2991943359375,
+      "logps/rejected": -389.4167175292969,
+      "loss": 0.0435,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4790451526641846,
+      "rewards/margins": 12.147565841674805,
+      "rewards/rejected": -13.626611709594727,
+      "step": 1390
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 10.118850215739602,
+      "learning_rate": 5.2158273381294966e-08,
+      "logits/chosen": -2.7540392875671387,
+      "logits/rejected": -2.6350178718566895,
+      "logps/chosen": -203.79840087890625,
+      "logps/rejected": -262.6423034667969,
+      "loss": 0.0881,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -5.214129447937012,
+      "rewards/margins": 8.150261878967285,
+      "rewards/rejected": -13.36439037322998,
+      "step": 1400
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 6.195639443759268,
+      "learning_rate": 4.856115107913669e-08,
+      "logits/chosen": -2.778777837753296,
+      "logits/rejected": -2.7979698181152344,
+      "logps/chosen": -248.45901489257812,
+      "logps/rejected": -357.4532470703125,
+      "loss": 0.06,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6782010197639465,
+      "rewards/margins": 11.35696792602539,
+      "rewards/rejected": -12.03516960144043,
+      "step": 1410
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 0.19531387796331173,
+      "learning_rate": 4.496402877697841e-08,
+      "logits/chosen": -2.6897778511047363,
+      "logits/rejected": -2.7521700859069824,
+      "logps/chosen": -278.0186767578125,
+      "logps/rejected": -327.90216064453125,
+      "loss": 0.0685,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -3.4422764778137207,
+      "rewards/margins": 11.567034721374512,
+      "rewards/rejected": -15.009310722351074,
+      "step": 1420
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 5.15723697089762,
+      "learning_rate": 4.136690647482014e-08,
+      "logits/chosen": -2.8552756309509277,
+      "logits/rejected": -2.815695285797119,
+      "logps/chosen": -189.77650451660156,
+      "logps/rejected": -303.08135986328125,
+      "loss": 0.0761,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.0464930534362793,
+      "rewards/margins": 10.03233528137207,
+      "rewards/rejected": -13.078828811645508,
+      "step": 1430
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.272902203913625,
+      "learning_rate": 3.776978417266187e-08,
+      "logits/chosen": -2.840439558029175,
+      "logits/rejected": -2.854123592376709,
+      "logps/chosen": -201.63998413085938,
+      "logps/rejected": -294.6650085449219,
+      "loss": 0.0943,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.1479772329330444,
+      "rewards/margins": 9.911928176879883,
+      "rewards/rejected": -11.059905052185059,
+      "step": 1440
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 6.709732675322476,
+      "learning_rate": 3.41726618705036e-08,
+      "logits/chosen": -2.6554994583129883,
+      "logits/rejected": -2.745185375213623,
+      "logps/chosen": -229.5808868408203,
+      "logps/rejected": -260.39434814453125,
+      "loss": 0.0925,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -4.566291809082031,
+      "rewards/margins": 6.469024658203125,
+      "rewards/rejected": -11.035317420959473,
+      "step": 1450
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 7.800521530829396,
+      "learning_rate": 3.057553956834532e-08,
+      "logits/chosen": -3.0627307891845703,
+      "logits/rejected": -2.9227752685546875,
+      "logps/chosen": -265.76641845703125,
+      "logps/rejected": -373.4715881347656,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.11531007289886475,
+      "rewards/margins": 10.523666381835938,
+      "rewards/rejected": -10.408354759216309,
+      "step": 1460
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 9.312558432834338,
+      "learning_rate": 2.6978417266187048e-08,
+      "logits/chosen": -3.0164947509765625,
+      "logits/rejected": -2.9149060249328613,
+      "logps/chosen": -299.1823425292969,
+      "logps/rejected": -357.61688232421875,
+      "loss": 0.0641,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.9149433374404907,
+      "rewards/margins": 11.019976615905762,
+      "rewards/rejected": -12.934918403625488,
+      "step": 1470
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 12.48672504880718,
+      "learning_rate": 2.3381294964028775e-08,
+      "logits/chosen": -2.6838114261627197,
+      "logits/rejected": -2.7183403968811035,
+      "logps/chosen": -177.66366577148438,
+      "logps/rejected": -297.00103759765625,
+      "loss": 0.0596,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.6696372032165527,
+      "rewards/margins": 10.785930633544922,
+      "rewards/rejected": -13.45556926727295,
+      "step": 1480
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 18.95745366827935,
+      "learning_rate": 1.9784172661870502e-08,
+      "logits/chosen": -2.6468663215637207,
+      "logits/rejected": -2.6348750591278076,
+      "logps/chosen": -230.4730224609375,
+      "logps/rejected": -255.1483612060547,
+      "loss": 0.0823,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.6124650835990906,
+      "rewards/margins": 10.041625022888184,
+      "rewards/rejected": -10.65408992767334,
+      "step": 1490
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 0.8016635311410285,
+      "learning_rate": 1.618705035971223e-08,
+      "logits/chosen": -2.8173935413360596,
+      "logits/rejected": -2.7464711666107178,
+      "logps/chosen": -202.07252502441406,
+      "logps/rejected": -323.95135498046875,
+      "loss": 0.0776,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -2.8145692348480225,
+      "rewards/margins": 11.602530479431152,
+      "rewards/rejected": -14.417098999023438,
+      "step": 1500
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 80.91582579930383,
+      "learning_rate": 1.2589928057553956e-08,
+      "logits/chosen": -2.9232335090637207,
+      "logits/rejected": -2.922365188598633,
+      "logps/chosen": -182.398681640625,
+      "logps/rejected": -370.7036437988281,
+      "loss": 0.0741,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.1800309121608734,
+      "rewards/margins": 13.427223205566406,
+      "rewards/rejected": -13.2471923828125,
+      "step": 1510
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 0.5505807027887548,
+      "learning_rate": 8.992805755395683e-09,
+      "logits/chosen": -2.819612979888916,
+      "logits/rejected": -2.773894786834717,
+      "logps/chosen": -236.72335815429688,
+      "logps/rejected": -311.35552978515625,
+      "loss": 0.084,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.4171246588230133,
+      "rewards/margins": 12.874395370483398,
+      "rewards/rejected": -13.291519165039062,
+      "step": 1520
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 10.305894823331844,
+      "learning_rate": 5.39568345323741e-09,
+      "logits/chosen": -2.8409948348999023,
+      "logits/rejected": -2.846924066543579,
+      "logps/chosen": -266.71209716796875,
+      "logps/rejected": -324.0392150878906,
+      "loss": 0.0839,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.8426433801651,
+      "rewards/margins": 10.864583969116211,
+      "rewards/rejected": -9.021939277648926,
+      "step": 1530
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 21.7857600856585,
+      "learning_rate": 1.7985611510791367e-09,
+      "logits/chosen": -2.7813632488250732,
+      "logits/rejected": -2.836905002593994,
+      "logps/chosen": -192.66018676757812,
+      "logps/rejected": -365.13128662109375,
+      "loss": 0.0741,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4063602387905121,
+      "rewards/margins": 11.652292251586914,
+      "rewards/rejected": -12.058652877807617,
+      "step": 1540
+    },
+    {
+      "epoch": 3.0,
+      "step": 1545,
+      "total_flos": 0.0,
+      "train_loss": 0.25373340110176973,
+      "train_runtime": 3870.3099,
+      "train_samples_per_second": 6.386,
+      "train_steps_per_second": 0.399
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1545,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}