Model save

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: uf-mistral-it-orpo-iopo-iter1-short
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/nlee28/lucky/runs/np12dcdu)
+# uf-mistral-it-orpo-iopo-iter1-short
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.4
+- Pytorch 2.1.2.post303
+- Datasets 2.18.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9995320542817033,
+    "total_flos": 0.0,
+    "train_loss": 1.0056083492571941,
+    "train_runtime": 16376.6673,
+    "train_samples": 34190,
+    "train_samples_per_second": 2.088,
+    "train_steps_per_second": 0.065
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.4"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ee238dcd7930f2b88a5fbb2e81e30ad2ade2fc267eca61de2cd5c8f87d84509
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:284c723e08c2078c7d9e9b387e4e3d66532415738d02d7494f88eadd13938723
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0136607453b64757981b7f54023a9136890043c4772bc0fb3ce658ac092a330
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9995320542817033,
+    "total_flos": 0.0,
+    "train_loss": 1.0056083492571941,
+    "train_runtime": 16376.6673,
+    "train_samples": 34190,
+    "train_samples_per_second": 2.088,
+    "train_steps_per_second": 0.065
+}

trainer_state.json

@@ -0,0 +1,1950 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9995320542817033,
+  "eval_steps": 500,
+  "global_step": 1068,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009358914365933552,
+      "grad_norm": 36.0,
+      "learning_rate": 4.672897196261682e-08,
+      "log_odds_chosen": 0.027360862120985985,
+      "log_odds_ratio": -0.7125600576400757,
+      "logits/chosen": -2.177670955657959,
+      "logits/rejected": -2.1592280864715576,
+      "logps/chosen": -0.49792352318763733,
+      "logps/rejected": -0.5029939413070679,
+      "loss": 1.2707,
+      "nll_loss": 1.1897716522216797,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.04979235306382179,
+      "rewards/margins": 0.0005070472252555192,
+      "rewards/rejected": -0.05029939487576485,
+      "step": 10
+    },
+    {
+      "epoch": 0.018717828731867104,
+      "grad_norm": 33.75,
+      "learning_rate": 9.345794392523364e-08,
+      "log_odds_chosen": 0.10758718103170395,
+      "log_odds_ratio": -0.683829128742218,
+      "logits/chosen": -2.206817865371704,
+      "logits/rejected": -2.195065975189209,
+      "logps/chosen": -0.5090934038162231,
+      "logps/rejected": -0.5577604174613953,
+      "loss": 1.3409,
+      "nll_loss": 1.2595471143722534,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.050909340381622314,
+      "rewards/margins": 0.0048667010851204395,
+      "rewards/rejected": -0.05577604100108147,
+      "step": 20
+    },
+    {
+      "epoch": 0.028076743097800654,
+      "grad_norm": 28.375,
+      "learning_rate": 1.4018691588785045e-07,
+      "log_odds_chosen": -0.06396888941526413,
+      "log_odds_ratio": -0.7616688013076782,
+      "logits/chosen": -2.1569924354553223,
+      "logits/rejected": -2.136850595474243,
+      "logps/chosen": -0.5089480876922607,
+      "logps/rejected": -0.48712658882141113,
+      "loss": 1.2937,
+      "nll_loss": 1.2157827615737915,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.05089481547474861,
+      "rewards/margins": -0.0021821539849042892,
+      "rewards/rejected": -0.04871265962719917,
+      "step": 30
+    },
+    {
+      "epoch": 0.03743565746373421,
+      "grad_norm": 33.5,
+      "learning_rate": 1.8691588785046729e-07,
+      "log_odds_chosen": 0.027819359675049782,
+      "log_odds_ratio": -0.7156058549880981,
+      "logits/chosen": -2.2016537189483643,
+      "logits/rejected": -2.1811304092407227,
+      "logps/chosen": -0.47956228256225586,
+      "logps/rejected": -0.48842015862464905,
+      "loss": 1.2882,
+      "nll_loss": 1.2497555017471313,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.047956231981515884,
+      "rewards/margins": 0.0008857922512106597,
+      "rewards/rejected": -0.048842012882232666,
+      "step": 40
+    },
+    {
+      "epoch": 0.04679457182966776,
+      "grad_norm": 26.125,
+      "learning_rate": 2.336448598130841e-07,
+      "log_odds_chosen": 0.07888893783092499,
+      "log_odds_ratio": -0.7044392824172974,
+      "logits/chosen": -2.106088161468506,
+      "logits/rejected": -2.102090358734131,
+      "logps/chosen": -0.4838789999485016,
+      "logps/rejected": -0.5239019989967346,
+      "loss": 1.2274,
+      "nll_loss": 1.1539485454559326,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.04838789626955986,
+      "rewards/margins": 0.004002300091087818,
+      "rewards/rejected": -0.0523901991546154,
+      "step": 50
+    },
+    {
+      "epoch": 0.05615348619560131,
+      "grad_norm": 29.625,
+      "learning_rate": 2.803738317757009e-07,
+      "log_odds_chosen": 0.0740145593881607,
+      "log_odds_ratio": -0.6914735436439514,
+      "logits/chosen": -2.1816258430480957,
+      "logits/rejected": -2.178445339202881,
+      "logps/chosen": -0.4614052176475525,
+      "logps/rejected": -0.4912230968475342,
+      "loss": 1.2751,
+      "nll_loss": 1.1571671962738037,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.046140529215335846,
+      "rewards/margins": 0.002981787081807852,
+      "rewards/rejected": -0.04912231117486954,
+      "step": 60
+    },
+    {
+      "epoch": 0.06551240056153486,
+      "grad_norm": 25.875,
+      "learning_rate": 3.271028037383177e-07,
+      "log_odds_chosen": -0.02471914514899254,
+      "log_odds_ratio": -0.741605818271637,
+      "logits/chosen": -2.2288427352905273,
+      "logits/rejected": -2.2079877853393555,
+      "logps/chosen": -0.48365911841392517,
+      "logps/rejected": -0.4822389483451843,
+      "loss": 1.3016,
+      "nll_loss": 1.2900102138519287,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.048365913331508636,
+      "rewards/margins": -0.00014201310114003718,
+      "rewards/rejected": -0.04822390154004097,
+      "step": 70
+    },
+    {
+      "epoch": 0.07487131492746842,
+      "grad_norm": 26.625,
+      "learning_rate": 3.7383177570093457e-07,
+      "log_odds_chosen": 0.11039517819881439,
+      "log_odds_ratio": -0.6913446187973022,
+      "logits/chosen": -2.2124581336975098,
+      "logits/rejected": -2.194650411605835,
+      "logps/chosen": -0.4804244935512543,
+      "logps/rejected": -0.5464648008346558,
+      "loss": 1.1872,
+      "nll_loss": 1.0913323163986206,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.04804245010018349,
+      "rewards/margins": 0.006604035850614309,
+      "rewards/rejected": -0.054646484553813934,
+      "step": 80
+    },
+    {
+      "epoch": 0.08423022929340196,
+      "grad_norm": 21.75,
+      "learning_rate": 4.205607476635514e-07,
+      "log_odds_chosen": -0.002868836745619774,
+      "log_odds_ratio": -0.7235916256904602,
+      "logits/chosen": -2.246835470199585,
+      "logits/rejected": -2.2278454303741455,
+      "logps/chosen": -0.4966287612915039,
+      "logps/rejected": -0.49866318702697754,
+      "loss": 1.2336,
+      "nll_loss": 1.1712753772735596,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.04966288059949875,
+      "rewards/margins": 0.00020344238146208227,
+      "rewards/rejected": -0.04986632615327835,
+      "step": 90
+    },
+    {
+      "epoch": 0.09358914365933552,
+      "grad_norm": 21.625,
+      "learning_rate": 4.672897196261682e-07,
+      "log_odds_chosen": 0.006027285009622574,
+      "log_odds_ratio": -0.7175697088241577,
+      "logits/chosen": -2.194366216659546,
+      "logits/rejected": -2.1875548362731934,
+      "logps/chosen": -0.4724898934364319,
+      "logps/rejected": -0.46772247552871704,
+      "loss": 1.1875,
+      "nll_loss": 1.1505517959594727,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.04724898934364319,
+      "rewards/margins": -0.00047674038796685636,
+      "rewards/rejected": -0.046772249042987823,
+      "step": 100
+    },
+    {
+      "epoch": 0.10294805802526907,
+      "grad_norm": 16.625,
+      "learning_rate": 4.999879772999679e-07,
+      "log_odds_chosen": 0.061622392386198044,
+      "log_odds_ratio": -0.6916904449462891,
+      "logits/chosen": -2.169373035430908,
+      "logits/rejected": -2.1729514598846436,
+      "logps/chosen": -0.4589506685733795,
+      "logps/rejected": -0.4915087819099426,
+      "loss": 1.1308,
+      "nll_loss": 1.0613527297973633,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.04589506983757019,
+      "rewards/margins": 0.003255808260291815,
+      "rewards/rejected": -0.04915087670087814,
+      "step": 110
+    },
+    {
+      "epoch": 0.11230697239120262,
+      "grad_norm": 15.75,
+      "learning_rate": 4.997742725777528e-07,
+      "log_odds_chosen": 0.06405100971460342,
+      "log_odds_ratio": -0.6904009580612183,
+      "logits/chosen": -2.2223846912384033,
+      "logits/rejected": -2.203917980194092,
+      "logps/chosen": -0.4754715859889984,
+      "logps/rejected": -0.5003005862236023,
+      "loss": 1.0866,
+      "nll_loss": 1.0029579401016235,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04754715785384178,
+      "rewards/margins": 0.0024829022586345673,
+      "rewards/rejected": -0.05003006383776665,
+      "step": 120
+    },
+    {
+      "epoch": 0.12166588675713617,
+      "grad_norm": 14.5625,
+      "learning_rate": 4.992936596071119e-07,
+      "log_odds_chosen": 0.08829396218061447,
+      "log_odds_ratio": -0.6891029477119446,
+      "logits/chosen": -2.222391128540039,
+      "logits/rejected": -2.2042291164398193,
+      "logps/chosen": -0.46681737899780273,
+      "logps/rejected": -0.5088747143745422,
+      "loss": 1.1225,
+      "nll_loss": 1.0937663316726685,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04668173938989639,
+      "rewards/margins": 0.004205727018415928,
+      "rewards/rejected": -0.05088747292757034,
+      "step": 130
+    },
+    {
+      "epoch": 0.13102480112306972,
+      "grad_norm": 15.3125,
+      "learning_rate": 4.985466519700276e-07,
+      "log_odds_chosen": 0.06562065333127975,
+      "log_odds_ratio": -0.6990588903427124,
+      "logits/chosen": -2.2474286556243896,
+      "logits/rejected": -2.2379136085510254,
+      "logps/chosen": -0.47886237502098083,
+      "logps/rejected": -0.5128284096717834,
+      "loss": 1.1017,
+      "nll_loss": 1.040492296218872,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.047886233776807785,
+      "rewards/margins": 0.0033966104965656996,
+      "rewards/rejected": -0.051282841712236404,
+      "step": 140
+    },
+    {
+      "epoch": 0.14038371548900327,
+      "grad_norm": 15.625,
+      "learning_rate": 4.975340479172484e-07,
+      "log_odds_chosen": 0.03810061141848564,
+      "log_odds_ratio": -0.7092625498771667,
+      "logits/chosen": -2.2372212409973145,
+      "logits/rejected": -2.206622362136841,
+      "logps/chosen": -0.4353889524936676,
+      "logps/rejected": -0.44863614439964294,
+      "loss": 0.9927,
+      "nll_loss": 0.9188264608383179,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.0435388945043087,
+      "rewards/margins": 0.0013247155584394932,
+      "rewards/rejected": -0.04486361891031265,
+      "step": 150
+    },
+    {
+      "epoch": 0.14974262985493683,
+      "grad_norm": 16.0,
+      "learning_rate": 4.962569295152789e-07,
+      "log_odds_chosen": 0.1309119015932083,
+      "log_odds_ratio": -0.666577935218811,
+      "logits/chosen": -2.1617026329040527,
+      "logits/rejected": -2.1651737689971924,
+      "logps/chosen": -0.46459126472473145,
+      "logps/rejected": -0.5068212747573853,
+      "loss": 1.1044,
+      "nll_loss": 1.0174779891967773,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.046459127217531204,
+      "rewards/margins": 0.004223003052175045,
+      "rewards/rejected": -0.050682127475738525,
+      "step": 160
+    },
+    {
+      "epoch": 0.1591015442208704,
+      "grad_norm": 16.0,
+      "learning_rate": 4.947166614900862e-07,
+      "log_odds_chosen": 0.05984800308942795,
+      "log_odds_ratio": -0.6976341009140015,
+      "logits/chosen": -2.2195987701416016,
+      "logits/rejected": -2.2141668796539307,
+      "logps/chosen": -0.4686553478240967,
+      "logps/rejected": -0.4932120740413666,
+      "loss": 0.9915,
+      "nll_loss": 0.9667671918869019,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04686553403735161,
+      "rewards/margins": 0.0024556724820286036,
+      "rewards/rejected": -0.04932120442390442,
+      "step": 170
+    },
+    {
+      "epoch": 0.16846045858680392,
+      "grad_norm": 15.75,
+      "learning_rate": 4.929148897687566e-07,
+      "log_odds_chosen": 0.2192097008228302,
+      "log_odds_ratio": -0.6804882884025574,
+      "logits/chosen": -2.2159080505371094,
+      "logits/rejected": -2.1967945098876953,
+      "logps/chosen": -0.45083823800086975,
+      "logps/rejected": -0.5973262786865234,
+      "loss": 0.9957,
+      "nll_loss": 0.9682543873786926,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04508382827043533,
+      "rewards/margins": 0.0146488007158041,
+      "rewards/rejected": -0.059732627123594284,
+      "step": 180
+    },
+    {
+      "epoch": 0.17781937295273748,
+      "grad_norm": 13.5625,
+      "learning_rate": 4.908535397206616e-07,
+      "log_odds_chosen": 0.1358589231967926,
+      "log_odds_ratio": -0.6608278751373291,
+      "logits/chosen": -2.1806671619415283,
+      "logits/rejected": -2.1832306385040283,
+      "logps/chosen": -0.42772555351257324,
+      "logps/rejected": -0.4777906835079193,
+      "loss": 1.0472,
+      "nll_loss": 0.9567255973815918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0427725613117218,
+      "rewards/margins": 0.005006510764360428,
+      "rewards/rejected": -0.04777907580137253,
+      "step": 190
+    },
+    {
+      "epoch": 0.18717828731867103,
+      "grad_norm": 13.9375,
+      "learning_rate": 4.885348141000122e-07,
+      "log_odds_chosen": 0.11417017132043839,
+      "log_odds_ratio": -0.6762548089027405,
+      "logits/chosen": -2.184654712677002,
+      "logits/rejected": -2.170086145401001,
+      "logps/chosen": -0.4621838629245758,
+      "logps/rejected": -0.518031120300293,
+      "loss": 1.0153,
+      "nll_loss": 0.9638531804084778,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.0462183877825737,
+      "rewards/margins": 0.00558472191914916,
+      "rewards/rejected": -0.0518031120300293,
+      "step": 200
+    },
+    {
+      "epoch": 0.1965372016846046,
+      "grad_norm": 14.3125,
+      "learning_rate": 4.859611906920007e-07,
+      "log_odds_chosen": 0.18621578812599182,
+      "log_odds_ratio": -0.6375190019607544,
+      "logits/chosen": -2.171351909637451,
+      "logits/rejected": -2.1605920791625977,
+      "logps/chosen": -0.43598541617393494,
+      "logps/rejected": -0.5046157240867615,
+      "loss": 1.0046,
+      "nll_loss": 0.995398998260498,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04359854385256767,
+      "rewards/margins": 0.006863036658614874,
+      "rewards/rejected": -0.050461579114198685,
+      "step": 210
+    },
+    {
+      "epoch": 0.20589611605053815,
+      "grad_norm": 14.9375,
+      "learning_rate": 4.831354196650446e-07,
+      "log_odds_chosen": 0.1494245082139969,
+      "log_odds_ratio": -0.6539293527603149,
+      "logits/chosen": -2.2000341415405273,
+      "logits/rejected": -2.1807079315185547,
+      "logps/chosen": -0.4382123053073883,
+      "logps/rejected": -0.4922669529914856,
+      "loss": 1.0051,
+      "nll_loss": 0.9585235714912415,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04382122680544853,
+      "rewards/margins": 0.005405460484325886,
+      "rewards/rejected": -0.04922669380903244,
+      "step": 220
+    },
+    {
+      "epoch": 0.21525503041647168,
+      "grad_norm": 14.25,
+      "learning_rate": 4.800605206319624e-07,
+      "log_odds_chosen": 0.19924063980579376,
+      "log_odds_ratio": -0.6384560465812683,
+      "logits/chosen": -2.1733033657073975,
+      "logits/rejected": -2.1657328605651855,
+      "logps/chosen": -0.40226203203201294,
+      "logps/rejected": -0.4800339639186859,
+      "loss": 0.9612,
+      "nll_loss": 0.872852623462677,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04022620618343353,
+      "rewards/margins": 0.007777197752147913,
+      "rewards/rejected": -0.04800339788198471,
+      "step": 230
+    },
+    {
+      "epoch": 0.22461394478240523,
+      "grad_norm": 14.4375,
+      "learning_rate": 4.767397794232225e-07,
+      "log_odds_chosen": 0.14606961607933044,
+      "log_odds_ratio": -0.652389645576477,
+      "logits/chosen": -2.217345714569092,
+      "logits/rejected": -2.193671703338623,
+      "logps/chosen": -0.4080810546875,
+      "logps/rejected": -0.459284245967865,
+      "loss": 0.9276,
+      "nll_loss": 0.856499195098877,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04080810770392418,
+      "rewards/margins": 0.005120322573930025,
+      "rewards/rejected": -0.045928433537483215,
+      "step": 240
+    },
+    {
+      "epoch": 0.2339728591483388,
+      "grad_norm": 14.0,
+      "learning_rate": 4.731767445757111e-07,
+      "log_odds_chosen": 0.1912066489458084,
+      "log_odds_ratio": -0.6586419343948364,
+      "logits/chosen": -2.1863677501678467,
+      "logits/rejected": -2.1660654544830322,
+      "logps/chosen": -0.4087969660758972,
+      "logps/rejected": -0.5012443661689758,
+      "loss": 1.0828,
+      "nll_loss": 0.9812431335449219,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04087969660758972,
+      "rewards/margins": 0.009244738146662712,
+      "rewards/rejected": -0.05012443661689758,
+      "step": 250
+    },
+    {
+      "epoch": 0.24333177351427235,
+      "grad_norm": 13.8125,
+      "learning_rate": 4.6937522354077397e-07,
+      "log_odds_chosen": 0.07517801970243454,
+      "log_odds_ratio": -0.6879104375839233,
+      "logits/chosen": -2.2382235527038574,
+      "logits/rejected": -2.2303104400634766,
+      "logps/chosen": -0.4272983968257904,
+      "logps/rejected": -0.45664095878601074,
+      "loss": 1.0369,
+      "nll_loss": 0.9628226161003113,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.04272983968257904,
+      "rewards/margins": 0.0029342600610107183,
+      "rewards/rejected": -0.04566409811377525,
+      "step": 260
+    },
+    {
+      "epoch": 0.2526906878802059,
+      "grad_norm": 14.375,
+      "learning_rate": 4.6533927861558166e-07,
+      "log_odds_chosen": 0.20296251773834229,
+      "log_odds_ratio": -0.6265792846679688,
+      "logits/chosen": -2.2084856033325195,
+      "logits/rejected": -2.191228151321411,
+      "logps/chosen": -0.4132423400878906,
+      "logps/rejected": -0.4871252179145813,
+      "loss": 1.0579,
+      "nll_loss": 0.9442154765129089,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.04132423549890518,
+      "rewards/margins": 0.007388289086520672,
+      "rewards/rejected": -0.04871252924203873,
+      "step": 270
+    },
+    {
+      "epoch": 0.26204960224613943,
+      "grad_norm": 17.375,
+      "learning_rate": 4.6107322260216787e-07,
+      "log_odds_chosen": 0.19398412108421326,
+      "log_odds_ratio": -0.6390923261642456,
+      "logits/chosen": -2.160511016845703,
+      "logits/rejected": -2.1479556560516357,
+      "logps/chosen": -0.3960379362106323,
+      "logps/rejected": -0.47004491090774536,
+      "loss": 0.9542,
+      "nll_loss": 0.8770105242729187,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03960379213094711,
+      "rewards/margins": 0.007400700356811285,
+      "rewards/rejected": -0.047004491090774536,
+      "step": 280
+    },
+    {
+      "epoch": 0.271408516612073,
+      "grad_norm": 15.25,
+      "learning_rate": 4.565816141987782e-07,
+      "log_odds_chosen": 0.27176782488822937,
+      "log_odds_ratio": -0.6160404682159424,
+      "logits/chosen": -2.181396961212158,
+      "logits/rejected": -2.1620774269104004,
+      "logps/chosen": -0.3963298499584198,
+      "logps/rejected": -0.5069735646247864,
+      "loss": 0.9935,
+      "nll_loss": 0.932470977306366,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03963298350572586,
+      "rewards/margins": 0.01106436736881733,
+      "rewards/rejected": -0.05069734901189804,
+      "step": 290
+    },
+    {
+      "epoch": 0.28076743097800655,
+      "grad_norm": 16.125,
+      "learning_rate": 4.518692531284555e-07,
+      "log_odds_chosen": 0.1647796332836151,
+      "log_odds_ratio": -0.6567476391792297,
+      "logits/chosen": -2.1729674339294434,
+      "logits/rejected": -2.177335023880005,
+      "logps/chosen": -0.43337172269821167,
+      "logps/rejected": -0.5012940168380737,
+      "loss": 1.0111,
+      "nll_loss": 0.9405239820480347,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.043337173759937286,
+      "rewards/margins": 0.006792229600250721,
+      "rewards/rejected": -0.05012940242886543,
+      "step": 300
+    },
+    {
+      "epoch": 0.2901263453439401,
+      "grad_norm": 15.875,
+      "learning_rate": 4.469411750100657e-07,
+      "log_odds_chosen": 0.20056188106536865,
+      "log_odds_ratio": -0.6289246678352356,
+      "logits/chosen": -2.1089398860931396,
+      "logits/rejected": -2.097630023956299,
+      "logps/chosen": -0.419585645198822,
+      "logps/rejected": -0.4954373240470886,
+      "loss": 1.0291,
+      "nll_loss": 0.9756892323493958,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04195856675505638,
+      "rewards/margins": 0.007585165090858936,
+      "rewards/rejected": -0.04954373463988304,
+      "step": 310
+    },
+    {
+      "epoch": 0.29948525970987366,
+      "grad_norm": 14.9375,
+      "learning_rate": 4.418026459772465e-07,
+      "log_odds_chosen": 0.2775348424911499,
+      "log_odds_ratio": -0.6147114038467407,
+      "logits/chosen": -2.151935577392578,
+      "logits/rejected": -2.149029016494751,
+      "logps/chosen": -0.39582258462905884,
+      "logps/rejected": -0.5231629610061646,
+      "loss": 0.9973,
+      "nll_loss": 0.8973016738891602,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.039582256227731705,
+      "rewards/margins": 0.012734037823975086,
+      "rewards/rejected": -0.052316296845674515,
+      "step": 320
+    },
+    {
+      "epoch": 0.3088441740758072,
+      "grad_norm": 13.125,
+      "learning_rate": 4.3645915705102875e-07,
+      "log_odds_chosen": 0.1660947948694229,
+      "log_odds_ratio": -0.650874674320221,
+      "logits/chosen": -2.193951368331909,
+      "logits/rejected": -2.201650619506836,
+      "logps/chosen": -0.4254066050052643,
+      "logps/rejected": -0.4931983947753906,
+      "loss": 1.0248,
+      "nll_loss": 0.9826960563659668,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04254066199064255,
+      "rewards/margins": 0.006779176648706198,
+      "rewards/rejected": -0.04931983724236488,
+      "step": 330
+    },
+    {
+      "epoch": 0.3182030884417408,
+      "grad_norm": 16.875,
+      "learning_rate": 4.3091641827214255e-07,
+      "log_odds_chosen": 0.23508663475513458,
+      "log_odds_ratio": -0.6352331042289734,
+      "logits/chosen": -2.1884193420410156,
+      "logits/rejected": -2.194756269454956,
+      "logps/chosen": -0.3882560729980469,
+      "logps/rejected": -0.4830912947654724,
+      "loss": 0.9394,
+      "nll_loss": 0.8814057111740112,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03882560878992081,
+      "rewards/margins": 0.0094835190102458,
+      "rewards/rejected": -0.04830912500619888,
+      "step": 340
+    },
+    {
+      "epoch": 0.32756200280767434,
+      "grad_norm": 14.375,
+      "learning_rate": 4.251803525992806e-07,
+      "log_odds_chosen": 0.16706052422523499,
+      "log_odds_ratio": -0.6470171213150024,
+      "logits/chosen": -2.176159620285034,
+      "logits/rejected": -2.174161911010742,
+      "logps/chosen": -0.4145967960357666,
+      "logps/rejected": -0.486347496509552,
+      "loss": 0.9638,
+      "nll_loss": 0.9122942686080933,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04145968705415726,
+      "rewards/margins": 0.00717506418004632,
+      "rewards/rejected": -0.04863474518060684,
+      "step": 350
+    },
+    {
+      "epoch": 0.33692091717360784,
+      "grad_norm": 16.5,
+      "learning_rate": 4.192570895798369e-07,
+      "log_odds_chosen": 0.23265504837036133,
+      "log_odds_ratio": -0.6187289357185364,
+      "logits/chosen": -2.1640992164611816,
+      "logits/rejected": -2.168822765350342,
+      "logps/chosen": -0.40140947699546814,
+      "logps/rejected": -0.4813784658908844,
+      "loss": 0.9937,
+      "nll_loss": 0.8817814588546753,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04014094918966293,
+      "rewards/margins": 0.007996905595064163,
+      "rewards/rejected": -0.0481378547847271,
+      "step": 360
+    },
+    {
+      "epoch": 0.3462798315395414,
+      "grad_norm": 16.75,
+      "learning_rate": 4.1315295879988603e-07,
+      "log_odds_chosen": 0.1851670742034912,
+      "log_odds_ratio": -0.641275942325592,
+      "logits/chosen": -2.17275333404541,
+      "logits/rejected": -2.1673264503479004,
+      "logps/chosen": -0.40128177404403687,
+      "logps/rejected": -0.4769595265388489,
+      "loss": 0.9637,
+      "nll_loss": 0.9042321443557739,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.040128178894519806,
+      "rewards/margins": 0.007567767985165119,
+      "rewards/rejected": -0.04769594967365265,
+      "step": 370
+    },
+    {
+      "epoch": 0.35563874590547495,
+      "grad_norm": 15.25,
+      "learning_rate": 4.06874483120401e-07,
+      "log_odds_chosen": 0.22965331375598907,
+      "log_odds_ratio": -0.6257021427154541,
+      "logits/chosen": -2.2078561782836914,
+      "logits/rejected": -2.190913677215576,
+      "logps/chosen": -0.39686036109924316,
+      "logps/rejected": -0.4888846278190613,
+      "loss": 1.0607,
+      "nll_loss": 0.9836522340774536,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03968603163957596,
+      "rewards/margins": 0.009202429093420506,
+      "rewards/rejected": -0.04888846352696419,
+      "step": 380
+    },
+    {
+      "epoch": 0.3649976602714085,
+      "grad_norm": 15.25,
+      "learning_rate": 4.00428371706938e-07,
+      "log_odds_chosen": 0.2075185775756836,
+      "log_odds_ratio": -0.624189019203186,
+      "logits/chosen": -2.219852924346924,
+      "logits/rejected": -2.1979775428771973,
+      "logps/chosen": -0.4044666886329651,
+      "logps/rejected": -0.4880141317844391,
+      "loss": 0.9671,
+      "nll_loss": 0.9435387849807739,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04044666886329651,
+      "rewards/margins": 0.008354747667908669,
+      "rewards/rejected": -0.04880141466856003,
+      "step": 390
+    },
+    {
+      "epoch": 0.37435657463734207,
+      "grad_norm": 16.25,
+      "learning_rate": 3.9382151286023644e-07,
+      "log_odds_chosen": 0.18440015614032745,
+      "log_odds_ratio": -0.6386502981185913,
+      "logits/chosen": -2.2246038913726807,
+      "logits/rejected": -2.222909450531006,
+      "logps/chosen": -0.4041818082332611,
+      "logps/rejected": -0.47141432762145996,
+      "loss": 0.915,
+      "nll_loss": 0.8205062747001648,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04041817784309387,
+      "rewards/margins": 0.006723250262439251,
+      "rewards/rejected": -0.0471414290368557,
+      "step": 400
+    },
+    {
+      "epoch": 0.3837154890032756,
+      "grad_norm": 16.0,
+      "learning_rate": 3.8706096665539617e-07,
+      "log_odds_chosen": 0.27819472551345825,
+      "log_odds_ratio": -0.628924548625946,
+      "logits/chosen": -2.2116692066192627,
+      "logits/rejected": -2.1926889419555664,
+      "logps/chosen": -0.39858338236808777,
+      "logps/rejected": -0.5448154211044312,
+      "loss": 1.0069,
+      "nll_loss": 0.9505244493484497,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.039858341217041016,
+      "rewards/margins": 0.014623197726905346,
+      "rewards/rejected": -0.054481543600559235,
+      "step": 410
+    },
+    {
+      "epoch": 0.3930744033692092,
+      "grad_norm": 14.125,
+      "learning_rate": 3.801539573974959e-07,
+      "log_odds_chosen": 0.3876788318157196,
+      "log_odds_ratio": -0.6021304130554199,
+      "logits/chosen": -2.2393152713775635,
+      "logits/rejected": -2.2127761840820312,
+      "logps/chosen": -0.3716931939125061,
+      "logps/rejected": -0.5833589434623718,
+      "loss": 0.9809,
+      "nll_loss": 0.9177610278129578,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03716932237148285,
+      "rewards/margins": 0.021166574209928513,
+      "rewards/rejected": -0.05833590030670166,
+      "step": 420
+    },
+    {
+      "epoch": 0.40243331773514274,
+      "grad_norm": 14.0,
+      "learning_rate": 3.7310786590171683e-07,
+      "log_odds_chosen": 0.18535861372947693,
+      "log_odds_ratio": -0.6408358812332153,
+      "logits/chosen": -2.23064923286438,
+      "logits/rejected": -2.2105820178985596,
+      "logps/chosen": -0.40638476610183716,
+      "logps/rejected": -0.4804498255252838,
+      "loss": 1.0212,
+      "nll_loss": 0.9341148138046265,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04063846915960312,
+      "rewards/margins": 0.007406515069305897,
+      "rewards/rejected": -0.04804498702287674,
+      "step": 430
+    },
+    {
+      "epoch": 0.4117922321010763,
+      "grad_norm": 23.625,
+      "learning_rate": 3.659302216062191e-07,
+      "log_odds_chosen": 0.2502005398273468,
+      "log_odds_ratio": -0.6098745465278625,
+      "logits/chosen": -2.180509567260742,
+      "logits/rejected": -2.174722671508789,
+      "logps/chosen": -0.39229241013526917,
+      "logps/rejected": -0.48466572165489197,
+      "loss": 0.9713,
+      "nll_loss": 0.9427000284194946,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03922923654317856,
+      "rewards/margins": 0.009237336926162243,
+      "rewards/rejected": -0.048466578125953674,
+      "step": 440
+    },
+    {
+      "epoch": 0.42115114646700985,
+      "grad_norm": 13.5625,
+      "learning_rate": 3.586286945262007e-07,
+      "log_odds_chosen": 0.2504832148551941,
+      "log_odds_ratio": -0.6089121103286743,
+      "logits/chosen": -2.0650932788848877,
+      "logits/rejected": -2.0757880210876465,
+      "logps/chosen": -0.3794267177581787,
+      "logps/rejected": -0.4683166444301605,
+      "loss": 0.9343,
+      "nll_loss": 0.8940143585205078,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03794267028570175,
+      "rewards/margins": 0.00888899713754654,
+      "rewards/rejected": -0.04683166742324829,
+      "step": 450
+    },
+    {
+      "epoch": 0.43051006083294335,
+      "grad_norm": 15.25,
+      "learning_rate": 3.512110870577357e-07,
+      "log_odds_chosen": 0.23200464248657227,
+      "log_odds_ratio": -0.6252235770225525,
+      "logits/chosen": -2.1500890254974365,
+      "logits/rejected": -2.1427481174468994,
+      "logps/chosen": -0.39273735880851746,
+      "logps/rejected": -0.4773070812225342,
+      "loss": 1.0269,
+      "nll_loss": 0.9092708826065063,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.039273738861083984,
+      "rewards/margins": 0.008456969633698463,
+      "rewards/rejected": -0.0477307066321373,
+      "step": 460
+    },
+    {
+      "epoch": 0.4398689751988769,
+      "grad_norm": 11.9375,
+      "learning_rate": 3.436853256401506e-07,
+      "log_odds_chosen": 0.23003438115119934,
+      "log_odds_ratio": -0.6194323301315308,
+      "logits/chosen": -2.1569676399230957,
+      "logits/rejected": -2.1662819385528564,
+      "logps/chosen": -0.3882262110710144,
+      "logps/rejected": -0.4661303162574768,
+      "loss": 0.9625,
+      "nll_loss": 0.8705393671989441,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.03882262483239174,
+      "rewards/margins": 0.007790408097207546,
+      "rewards/rejected": -0.04661303386092186,
+      "step": 470
+    },
+    {
+      "epoch": 0.44922788956481047,
+      "grad_norm": 15.5,
+      "learning_rate": 3.3605945228584866e-07,
+      "log_odds_chosen": 0.1848507523536682,
+      "log_odds_ratio": -0.637083113193512,
+      "logits/chosen": -2.208573579788208,
+      "logits/rejected": -2.1971964836120605,
+      "logps/chosen": -0.4037590026855469,
+      "logps/rejected": -0.473149836063385,
+      "loss": 0.9891,
+      "nll_loss": 0.9415995478630066,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04037589952349663,
+      "rewards/margins": 0.006939084734767675,
+      "rewards/rejected": -0.04731498286128044,
+      "step": 480
+    },
+    {
+      "epoch": 0.458586803930744,
+      "grad_norm": 13.0625,
+      "learning_rate": 3.283416159866321e-07,
+      "log_odds_chosen": 0.24159303307533264,
+      "log_odds_ratio": -0.6120076179504395,
+      "logits/chosen": -2.184213161468506,
+      "logits/rejected": -2.1987740993499756,
+      "logps/chosen": -0.39372795820236206,
+      "logps/rejected": -0.48281997442245483,
+      "loss": 0.9589,
+      "nll_loss": 0.8757813572883606,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.039372798055410385,
+      "rewards/margins": 0.00890920776873827,
+      "rewards/rejected": -0.04828200489282608,
+      "step": 490
+    },
+    {
+      "epoch": 0.4679457182966776,
+      "grad_norm": 16.375,
+      "learning_rate": 3.205400640057081e-07,
+      "log_odds_chosen": 0.133102148771286,
+      "log_odds_ratio": -0.6767079830169678,
+      "logits/chosen": -2.183706760406494,
+      "logits/rejected": -2.1818044185638428,
+      "logps/chosen": -0.4044373631477356,
+      "logps/rejected": -0.4518662095069885,
+      "loss": 1.0155,
+      "nll_loss": 0.9624871015548706,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04044373333454132,
+      "rewards/margins": 0.004742881283164024,
+      "rewards/rejected": -0.045186616480350494,
+      "step": 500
+    },
+    {
+      "epoch": 0.47730463266261114,
+      "grad_norm": 15.5625,
+      "learning_rate": 3.126631330646801e-07,
+      "log_odds_chosen": 0.2868886888027191,
+      "log_odds_ratio": -0.6187784671783447,
+      "logits/chosen": -2.2198147773742676,
+      "logits/rejected": -2.224412441253662,
+      "logps/chosen": -0.3866862952709198,
+      "logps/rejected": -0.5035656094551086,
+      "loss": 0.9648,
+      "nll_loss": 0.8982638120651245,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03866862505674362,
+      "rewards/margins": 0.01168792974203825,
+      "rewards/rejected": -0.050356559455394745,
+      "step": 510
+    },
+    {
+      "epoch": 0.4866635470285447,
+      "grad_norm": 14.125,
+      "learning_rate": 3.0471924043494595e-07,
+      "log_odds_chosen": 0.24677081406116486,
+      "log_odds_ratio": -0.618061363697052,
+      "logits/chosen": -2.144286870956421,
+      "logits/rejected": -2.1702799797058105,
+      "logps/chosen": -0.3944970667362213,
+      "logps/rejected": -0.49615901708602905,
+      "loss": 0.9674,
+      "nll_loss": 0.8812870979309082,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03944971039891243,
+      "rewards/margins": 0.010166196152567863,
+      "rewards/rejected": -0.049615900963544846,
+      "step": 520
+    },
+    {
+      "epoch": 0.49602246139447825,
+      "grad_norm": 13.0,
+      "learning_rate": 2.967168749430191e-07,
+      "log_odds_chosen": 0.25857609510421753,
+      "log_odds_ratio": -0.6192947030067444,
+      "logits/chosen": -2.184696674346924,
+      "logits/rejected": -2.1700081825256348,
+      "logps/chosen": -0.3798629939556122,
+      "logps/rejected": -0.49370065331459045,
+      "loss": 0.9305,
+      "nll_loss": 0.8286808133125305,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03798630088567734,
+      "rewards/margins": 0.011383764445781708,
+      "rewards/rejected": -0.04937007278203964,
+      "step": 530
+    },
+    {
+      "epoch": 0.5053813757604118,
+      "grad_norm": 15.3125,
+      "learning_rate": 2.8866458789938774e-07,
+      "log_odds_chosen": 0.3041766285896301,
+      "log_odds_ratio": -0.6016188859939575,
+      "logits/chosen": -2.1878724098205566,
+      "logits/rejected": -2.197441577911377,
+      "logps/chosen": -0.3805080056190491,
+      "logps/rejected": -0.5073161721229553,
+      "loss": 0.9574,
+      "nll_loss": 0.8998135328292847,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.038050804287195206,
+      "rewards/margins": 0.012680811807513237,
+      "rewards/rejected": -0.050731610506772995,
+      "step": 540
+    },
+    {
+      "epoch": 0.5147402901263454,
+      "grad_norm": 13.875,
+      "learning_rate": 2.8057098396060196e-07,
+      "log_odds_chosen": 0.13988998532295227,
+      "log_odds_ratio": -0.665267825126648,
+      "logits/chosen": -2.1989827156066895,
+      "logits/rejected": -2.184476852416992,
+      "logps/chosen": -0.3995954096317291,
+      "logps/rejected": -0.4471829831600189,
+      "loss": 0.9386,
+      "nll_loss": 0.8581770062446594,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.039959538727998734,
+      "rewards/margins": 0.004758755676448345,
+      "rewards/rejected": -0.04471829533576965,
+      "step": 550
+    },
+    {
+      "epoch": 0.5240992044922789,
+      "grad_norm": 16.375,
+      "learning_rate": 2.724447119343572e-07,
+      "log_odds_chosen": 0.209095761179924,
+      "log_odds_ratio": -0.6507378816604614,
+      "logits/chosen": -2.19320011138916,
+      "logits/rejected": -2.1836113929748535,
+      "logps/chosen": -0.39416733384132385,
+      "logps/rejected": -0.4946066439151764,
+      "loss": 0.9738,
+      "nll_loss": 0.9462787508964539,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.039416730403900146,
+      "rewards/margins": 0.010043924674391747,
+      "rewards/rejected": -0.04946065694093704,
+      "step": 560
+    },
+    {
+      "epoch": 0.5334581188582125,
+      "grad_norm": 14.5625,
+      "learning_rate": 2.642944555373965e-07,
+      "log_odds_chosen": 0.20115435123443604,
+      "log_odds_ratio": -0.6333853006362915,
+      "logits/chosen": -2.2073943614959717,
+      "logits/rejected": -2.212639093399048,
+      "logps/chosen": -0.39787840843200684,
+      "logps/rejected": -0.47424665093421936,
+      "loss": 0.9612,
+      "nll_loss": 0.9760659337043762,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03978784382343292,
+      "rewards/margins": 0.007636821363121271,
+      "rewards/rejected": -0.04742466285824776,
+      "step": 570
+    },
+    {
+      "epoch": 0.542817033224146,
+      "grad_norm": 15.625,
+      "learning_rate": 2.561289241161095e-07,
+      "log_odds_chosen": 0.2170640528202057,
+      "log_odds_ratio": -0.6275693774223328,
+      "logits/chosen": -2.196486234664917,
+      "logits/rejected": -2.1990580558776855,
+      "logps/chosen": -0.3897285461425781,
+      "logps/rejected": -0.4585072100162506,
+      "loss": 1.0314,
+      "nll_loss": 0.9672943949699402,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03897285833954811,
+      "rewards/margins": 0.006877865642309189,
+      "rewards/rejected": -0.0458507239818573,
+      "step": 580
+    },
+    {
+      "epoch": 0.5521759475900796,
+      "grad_norm": 14.0625,
+      "learning_rate": 2.479568433397441e-07,
+      "log_odds_chosen": 0.19771938025951385,
+      "log_odds_ratio": -0.6418076753616333,
+      "logits/chosen": -2.196673631668091,
+      "logits/rejected": -2.1924281120300293,
+      "logps/chosen": -0.3822599947452545,
+      "logps/rejected": -0.4602930545806885,
+      "loss": 0.9235,
+      "nll_loss": 0.8624385595321655,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03822600096464157,
+      "rewards/margins": 0.007803303189575672,
+      "rewards/rejected": -0.04602930322289467,
+      "step": 590
+    },
+    {
+      "epoch": 0.5615348619560131,
+      "grad_norm": 16.375,
+      "learning_rate": 2.3978694587617473e-07,
+      "log_odds_chosen": 0.24181723594665527,
+      "log_odds_ratio": -0.6263023614883423,
+      "logits/chosen": -2.1748032569885254,
+      "logits/rejected": -2.196155309677124,
+      "logps/chosen": -0.3913511335849762,
+      "logps/rejected": -0.47442513704299927,
+      "loss": 0.9472,
+      "nll_loss": 0.8652606010437012,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03913511335849762,
+      "rewards/margins": 0.008307400159537792,
+      "rewards/rejected": -0.047442514449357986,
+      "step": 600
+    },
+    {
+      "epoch": 0.5708937763219466,
+      "grad_norm": 15.6875,
+      "learning_rate": 2.3162796206019266e-07,
+      "log_odds_chosen": 0.25003498792648315,
+      "log_odds_ratio": -0.6166588068008423,
+      "logits/chosen": -2.122239351272583,
+      "logits/rejected": -2.120157241821289,
+      "logps/chosen": -0.39171507954597473,
+      "logps/rejected": -0.4818199574947357,
+      "loss": 0.9285,
+      "nll_loss": 0.812226414680481,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.039171505719423294,
+      "rewards/margins": 0.009010489098727703,
+      "rewards/rejected": -0.04818199574947357,
+      "step": 610
+    },
+    {
+      "epoch": 0.5802526906878802,
+      "grad_norm": 17.25,
+      "learning_rate": 2.2348861056428868e-07,
+      "log_odds_chosen": 0.23789414763450623,
+      "log_odds_ratio": -0.6213194131851196,
+      "logits/chosen": -2.2460482120513916,
+      "logits/rejected": -2.226989269256592,
+      "logps/chosen": -0.3984091281890869,
+      "logps/rejected": -0.5115697979927063,
+      "loss": 0.9339,
+      "nll_loss": 0.8749570846557617,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03984091058373451,
+      "rewards/margins": 0.011316070333123207,
+      "rewards/rejected": -0.05115697905421257,
+      "step": 620
+    },
+    {
+      "epoch": 0.5896116050538137,
+      "grad_norm": 17.0,
+      "learning_rate": 2.153775890818989e-07,
+      "log_odds_chosen": 0.22108840942382812,
+      "log_odds_ratio": -0.6147779822349548,
+      "logits/chosen": -2.239152431488037,
+      "logits/rejected": -2.2272543907165527,
+      "logps/chosen": -0.39232850074768066,
+      "logps/rejected": -0.46175652742385864,
+      "loss": 0.9649,
+      "nll_loss": 0.9323352575302124,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.039232850074768066,
+      "rewards/margins": 0.006942803505808115,
+      "rewards/rejected": -0.046175651252269745,
+      "step": 630
+    },
+    {
+      "epoch": 0.5989705194197473,
+      "grad_norm": 15.0,
+      "learning_rate": 2.0730356503306806e-07,
+      "log_odds_chosen": 0.12334088981151581,
+      "log_odds_ratio": -0.6655514240264893,
+      "logits/chosen": -2.1896536350250244,
+      "logits/rejected": -2.1869468688964844,
+      "logps/chosen": -0.3938687741756439,
+      "logps/rejected": -0.4362581670284271,
+      "loss": 0.9508,
+      "nll_loss": 0.8674702644348145,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.03938687965273857,
+      "rewards/margins": 0.0042389435693621635,
+      "rewards/rejected": -0.04362582042813301,
+      "step": 640
+    },
+    {
+      "epoch": 0.6083294337856808,
+      "grad_norm": 16.625,
+      "learning_rate": 1.9927516630246335e-07,
+      "log_odds_chosen": 0.20489105582237244,
+      "log_odds_ratio": -0.6253317594528198,
+      "logits/chosen": -2.2503440380096436,
+      "logits/rejected": -2.2342848777770996,
+      "logps/chosen": -0.394917756319046,
+      "logps/rejected": -0.46595415472984314,
+      "loss": 0.9924,
+      "nll_loss": 0.8516713976860046,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03949177637696266,
+      "rewards/margins": 0.0071036433801054955,
+      "rewards/rejected": -0.04659541696310043,
+      "step": 650
+    },
+    {
+      "epoch": 0.6176883481516144,
+      "grad_norm": 13.3125,
+      "learning_rate": 1.9130097201963545e-07,
+      "log_odds_chosen": 0.3604734539985657,
+      "log_odds_ratio": -0.5889347195625305,
+      "logits/chosen": -2.1605143547058105,
+      "logits/rejected": -2.1294660568237305,
+      "logps/chosen": -0.3886405825614929,
+      "logps/rejected": -0.5594497919082642,
+      "loss": 0.9053,
+      "nll_loss": 0.8217363357543945,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03886405751109123,
+      "rewards/margins": 0.017080921679735184,
+      "rewards/rejected": -0.055944979190826416,
+      "step": 660
+    },
+    {
+      "epoch": 0.6270472625175479,
+      "grad_norm": 15.125,
+      "learning_rate": 1.833895033913789e-07,
+      "log_odds_chosen": 0.3233751654624939,
+      "log_odds_ratio": -0.5963612794876099,
+      "logits/chosen": -2.1719839572906494,
+      "logits/rejected": -2.190833568572998,
+      "logps/chosen": -0.3922678530216217,
+      "logps/rejected": -0.535617470741272,
+      "loss": 0.9524,
+      "nll_loss": 0.9001427888870239,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.03922678157687187,
+      "rewards/margins": 0.014334957115352154,
+      "rewards/rejected": -0.0535617396235466,
+      "step": 670
+    },
+    {
+      "epoch": 0.6364061768834816,
+      "grad_norm": 15.875,
+      "learning_rate": 1.755492145959896e-07,
+      "log_odds_chosen": 0.31382274627685547,
+      "log_odds_ratio": -0.5895714163780212,
+      "logits/chosen": -2.211315155029297,
+      "logits/rejected": -2.233793020248413,
+      "logps/chosen": -0.3794475197792053,
+      "logps/rejected": -0.5004490613937378,
+      "loss": 0.9666,
+      "nll_loss": 0.919101893901825,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03794475272297859,
+      "rewards/margins": 0.012100161984562874,
+      "rewards/rejected": -0.050044916570186615,
+      "step": 680
+    },
+    {
+      "epoch": 0.6457650912494151,
+      "grad_norm": 14.25,
+      "learning_rate": 1.6778848374914728e-07,
+      "log_odds_chosen": 0.22027654945850372,
+      "log_odds_ratio": -0.6377453207969666,
+      "logits/chosen": -2.2121682167053223,
+      "logits/rejected": -2.1823742389678955,
+      "logps/chosen": -0.3827424645423889,
+      "logps/rejected": -0.4631470739841461,
+      "loss": 0.9201,
+      "nll_loss": 0.8197261691093445,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.03827425092458725,
+      "rewards/margins": 0.00804046168923378,
+      "rewards/rejected": -0.04631470888853073,
+      "step": 690
+    },
+    {
+      "epoch": 0.6551240056153487,
+      "grad_norm": 14.1875,
+      "learning_rate": 1.6011560395107998e-07,
+      "log_odds_chosen": 0.2072620838880539,
+      "log_odds_ratio": -0.6260396838188171,
+      "logits/chosen": -2.1534218788146973,
+      "logits/rejected": -2.135467052459717,
+      "logps/chosen": -0.4003871977329254,
+      "logps/rejected": -0.48006024956703186,
+      "loss": 0.9605,
+      "nll_loss": 0.9064604043960571,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04003872722387314,
+      "rewards/margins": 0.007967302575707436,
+      "rewards/rejected": -0.04800602048635483,
+      "step": 700
+    },
+    {
+      "epoch": 0.6644829199812822,
+      "grad_norm": 15.0625,
+      "learning_rate": 1.5253877442457446e-07,
+      "log_odds_chosen": 0.2182326316833496,
+      "log_odds_ratio": -0.6246551275253296,
+      "logits/chosen": -2.1990370750427246,
+      "logits/rejected": -2.187234878540039,
+      "logps/chosen": -0.39677727222442627,
+      "logps/rejected": -0.4731627404689789,
+      "loss": 0.9649,
+      "nll_loss": 0.8980569839477539,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.03967772796750069,
+      "rewards/margins": 0.007638547569513321,
+      "rewards/rejected": -0.04731627553701401,
+      "step": 710
+    },
+    {
+      "epoch": 0.6738418343472157,
+      "grad_norm": 14.625,
+      "learning_rate": 1.450660917533048e-07,
+      "log_odds_chosen": 0.2397657334804535,
+      "log_odds_ratio": -0.6196783185005188,
+      "logits/chosen": -2.1318721771240234,
+      "logits/rejected": -2.14440655708313,
+      "logps/chosen": -0.39654839038848877,
+      "logps/rejected": -0.49670910835266113,
+      "loss": 0.9984,
+      "nll_loss": 0.9670102000236511,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.039654843509197235,
+      "rewards/margins": 0.010016070678830147,
+      "rewards/rejected": -0.049670904874801636,
+      "step": 720
+    },
+    {
+      "epoch": 0.6832007487131493,
+      "grad_norm": 15.875,
+      "learning_rate": 1.377055412298402e-07,
+      "log_odds_chosen": 0.2634773552417755,
+      "log_odds_ratio": -0.6008420586585999,
+      "logits/chosen": -2.2238945960998535,
+      "logits/rejected": -2.2294325828552246,
+      "logps/chosen": -0.3819832503795624,
+      "logps/rejected": -0.48257875442504883,
+      "loss": 0.9467,
+      "nll_loss": 0.8946998715400696,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.038198329508304596,
+      "rewards/margins": 0.010059546679258347,
+      "rewards/rejected": -0.04825787618756294,
+      "step": 730
+    },
+    {
+      "epoch": 0.6925596630790828,
+      "grad_norm": 13.625,
+      "learning_rate": 1.3046498832257924e-07,
+      "log_odds_chosen": 0.37252184748649597,
+      "log_odds_ratio": -0.5736531019210815,
+      "logits/chosen": -2.1820709705352783,
+      "logits/rejected": -2.193258047103882,
+      "logps/chosen": -0.38064131140708923,
+      "logps/rejected": -0.529471218585968,
+      "loss": 0.9604,
+      "nll_loss": 0.8576000332832336,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.038064129650592804,
+      "rewards/margins": 0.014882983639836311,
+      "rewards/rejected": -0.052947115153074265,
+      "step": 740
+    },
+    {
+      "epoch": 0.7019185774450164,
+      "grad_norm": 18.25,
+      "learning_rate": 1.233521702707264e-07,
+      "log_odds_chosen": 0.20452013611793518,
+      "log_odds_ratio": -0.6333576440811157,
+      "logits/chosen": -2.2299623489379883,
+      "logits/rejected": -2.227400064468384,
+      "logps/chosen": -0.4053714871406555,
+      "logps/rejected": -0.4876191020011902,
+      "loss": 0.9534,
+      "nll_loss": 0.9184444546699524,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04053715243935585,
+      "rewards/margins": 0.008224757388234138,
+      "rewards/rejected": -0.04876190796494484,
+      "step": 750
+    },
+    {
+      "epoch": 0.7112774918109499,
+      "grad_norm": 12.0625,
+      "learning_rate": 1.1637468781629567e-07,
+      "log_odds_chosen": 0.2872607111930847,
+      "log_odds_ratio": -0.6115022897720337,
+      "logits/chosen": -2.1839165687561035,
+      "logits/rejected": -2.183589220046997,
+      "logps/chosen": -0.38220152258872986,
+      "logps/rejected": -0.5047668218612671,
+      "loss": 0.9557,
+      "nll_loss": 0.8811558485031128,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.03822015970945358,
+      "rewards/margins": 0.012256531044840813,
+      "rewards/rejected": -0.050476688891649246,
+      "step": 760
+    },
+    {
+      "epoch": 0.7206364061768835,
+      "grad_norm": 13.375,
+      "learning_rate": 1.0953999708197404e-07,
+      "log_odds_chosen": 0.2062472403049469,
+      "log_odds_ratio": -0.6398475766181946,
+      "logits/chosen": -2.2169864177703857,
+      "logits/rejected": -2.220945119857788,
+      "logps/chosen": -0.3971841633319855,
+      "logps/rejected": -0.48631978034973145,
+      "loss": 0.9464,
+      "nll_loss": 0.9025236368179321,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03971841558814049,
+      "rewards/margins": 0.00891356635838747,
+      "rewards/rejected": -0.048631977289915085,
+      "step": 770
+    },
+    {
+      "epoch": 0.729995320542817,
+      "grad_norm": 13.25,
+      "learning_rate": 1.0285540160352404e-07,
+      "log_odds_chosen": 0.18557827174663544,
+      "log_odds_ratio": -0.6399238705635071,
+      "logits/chosen": -2.2538866996765137,
+      "logits/rejected": -2.241267442703247,
+      "logps/chosen": -0.4030405879020691,
+      "logps/rejected": -0.47014960646629333,
+      "loss": 0.9506,
+      "nll_loss": 0.9130373001098633,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04030405730009079,
+      "rewards/margins": 0.006710907910019159,
+      "rewards/rejected": -0.04701497033238411,
+      "step": 780
+    },
+    {
+      "epoch": 0.7393542349087506,
+      "grad_norm": 13.375,
+      "learning_rate": 9.632804452524256e-08,
+      "log_odds_chosen": 0.2168750762939453,
+      "log_odds_ratio": -0.6271758675575256,
+      "logits/chosen": -2.145925760269165,
+      "logits/rejected": -2.147688865661621,
+      "logps/chosen": -0.40305909514427185,
+      "logps/rejected": -0.48798856139183044,
+      "loss": 0.8831,
+      "nll_loss": 0.8791316747665405,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.040305912494659424,
+      "rewards/margins": 0.008492941036820412,
+      "rewards/rejected": -0.048798851668834686,
+      "step": 790
+    },
+    {
+      "epoch": 0.7487131492746841,
+      "grad_norm": 13.6875,
+      "learning_rate": 8.996490096681109e-08,
+      "log_odds_chosen": 0.19949845969676971,
+      "log_odds_ratio": -0.6497266888618469,
+      "logits/chosen": -2.161623954772949,
+      "logits/rejected": -2.14546275138855,
+      "logps/chosen": -0.41016340255737305,
+      "logps/rejected": -0.49314770102500916,
+      "loss": 0.9495,
+      "nll_loss": 0.9498335123062134,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0410163477063179,
+      "rewards/margins": 0.00829843245446682,
+      "rewards/rejected": -0.049314774572849274,
+      "step": 800
+    },
+    {
+      "epoch": 0.7580720636406177,
+      "grad_norm": 13.875,
+      "learning_rate": 8.377277056969842e-08,
+      "log_odds_chosen": 0.2793668508529663,
+      "log_odds_ratio": -0.6106365323066711,
+      "logits/chosen": -2.225426435470581,
+      "logits/rejected": -2.216858386993408,
+      "logps/chosen": -0.40902504324913025,
+      "logps/rejected": -0.5241647958755493,
+      "loss": 0.9711,
+      "nll_loss": 0.9040560722351074,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.040902502834796906,
+      "rewards/margins": 0.01151397917419672,
+      "rewards/rejected": -0.05241648107767105,
+      "step": 810
+    },
+    {
+      "epoch": 0.7674309780065512,
+      "grad_norm": 16.125,
+      "learning_rate": 7.775827023107834e-08,
+      "log_odds_chosen": 0.29139918088912964,
+      "log_odds_ratio": -0.5888545513153076,
+      "logits/chosen": -2.2389183044433594,
+      "logits/rejected": -2.2176356315612793,
+      "logps/chosen": -0.39003506302833557,
+      "logps/rejected": -0.4982146620750427,
+      "loss": 0.9545,
+      "nll_loss": 0.9721220135688782,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03900350257754326,
+      "rewards/margins": 0.01081795897334814,
+      "rewards/rejected": -0.04982146620750427,
+      "step": 820
+    },
+    {
+      "epoch": 0.7767898923724847,
+      "grad_norm": 17.25,
+      "learning_rate": 7.192782703302785e-08,
+      "log_odds_chosen": 0.3136988878250122,
+      "log_odds_ratio": -0.5841724276542664,
+      "logits/chosen": -2.1989166736602783,
+      "logits/rejected": -2.1767630577087402,
+      "logps/chosen": -0.38346153497695923,
+      "logps/rejected": -0.49425944685935974,
+      "loss": 0.9185,
+      "nll_loss": 0.8561797142028809,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.03834615275263786,
+      "rewards/margins": 0.01107979379594326,
+      "rewards/rejected": -0.04942594841122627,
+      "step": 830
+    },
+    {
+      "epoch": 0.7861488067384184,
+      "grad_norm": 12.5625,
+      "learning_rate": 6.628767137456067e-08,
+      "log_odds_chosen": 0.20183363556861877,
+      "log_odds_ratio": -0.6364968419075012,
+      "logits/chosen": -2.163003444671631,
+      "logits/rejected": -2.169506788253784,
+      "logps/chosen": -0.4223906099796295,
+      "logps/rejected": -0.4997871518135071,
+      "loss": 0.9932,
+      "nll_loss": 0.9328519701957703,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04223906248807907,
+      "rewards/margins": 0.007739657070487738,
+      "rewards/rejected": -0.04997871816158295,
+      "step": 840
+    },
+    {
+      "epoch": 0.7955077211043519,
+      "grad_norm": 16.25,
+      "learning_rate": 6.08438303138365e-08,
+      "log_odds_chosen": 0.15886886417865753,
+      "log_odds_ratio": -0.6480736136436462,
+      "logits/chosen": -2.1696417331695557,
+      "logits/rejected": -2.1737618446350098,
+      "logps/chosen": -0.38803622126579285,
+      "logps/rejected": -0.44714784622192383,
+      "loss": 0.971,
+      "nll_loss": 0.917407214641571,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.038803622126579285,
+      "rewards/margins": 0.005911166779696941,
+      "rewards/rejected": -0.0447147861123085,
+      "step": 850
+    },
+    {
+      "epoch": 0.8048666354702855,
+      "grad_norm": 13.125,
+      "learning_rate": 5.560212112766011e-08,
+      "log_odds_chosen": 0.19609014689922333,
+      "log_odds_ratio": -0.6415227651596069,
+      "logits/chosen": -2.193361759185791,
+      "logits/rejected": -2.1855015754699707,
+      "logps/chosen": -0.3944851756095886,
+      "logps/rejected": -0.473043292760849,
+      "loss": 0.9432,
+      "nll_loss": 0.9007611274719238,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03944852203130722,
+      "rewards/margins": 0.007855813950300217,
+      "rewards/rejected": -0.04730432480573654,
+      "step": 860
+    },
+    {
+      "epoch": 0.814225549836219,
+      "grad_norm": 15.5625,
+      "learning_rate": 5.056814509515092e-08,
+      "log_odds_chosen": 0.27595824003219604,
+      "log_odds_ratio": -0.618240237236023,
+      "logits/chosen": -2.1721434593200684,
+      "logits/rejected": -2.2051827907562256,
+      "logps/chosen": -0.38330593705177307,
+      "logps/rejected": -0.4931607246398926,
+      "loss": 0.9415,
+      "nll_loss": 0.831408679485321,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03833059221506119,
+      "rewards/margins": 0.010985485278069973,
+      "rewards/rejected": -0.049316078424453735,
+      "step": 870
+    },
+    {
+      "epoch": 0.8235844642021526,
+      "grad_norm": 15.6875,
+      "learning_rate": 4.57472815122294e-08,
+      "log_odds_chosen": 0.1982923001050949,
+      "log_odds_ratio": -0.6450881958007812,
+      "logits/chosen": -2.2192018032073975,
+      "logits/rejected": -2.205094337463379,
+      "logps/chosen": -0.40160322189331055,
+      "logps/rejected": -0.4759804308414459,
+      "loss": 0.9264,
+      "nll_loss": 0.87147057056427,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.040160324424505234,
+      "rewards/margins": 0.007437719497829676,
+      "rewards/rejected": -0.04759804531931877,
+      "step": 880
+    },
+    {
+      "epoch": 0.8329433785680861,
+      "grad_norm": 14.5625,
+      "learning_rate": 4.1144681943312135e-08,
+      "log_odds_chosen": 0.26552319526672363,
+      "log_odds_ratio": -0.6161493062973022,
+      "logits/chosen": -2.134911298751831,
+      "logits/rejected": -2.146639347076416,
+      "logps/chosen": -0.3881653845310211,
+      "logps/rejected": -0.5064771771430969,
+      "loss": 0.9764,
+      "nll_loss": 0.9315429925918579,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03881654143333435,
+      "rewards/margins": 0.011831178329885006,
+      "rewards/rejected": -0.05064771696925163,
+      "step": 890
+    },
+    {
+      "epoch": 0.8423022929340197,
+      "grad_norm": 14.0,
+      "learning_rate": 3.676526471636168e-08,
+      "log_odds_chosen": 0.2872588634490967,
+      "log_odds_ratio": -0.6009818315505981,
+      "logits/chosen": -2.1846354007720947,
+      "logits/rejected": -2.1831088066101074,
+      "logps/chosen": -0.39365488290786743,
+      "logps/rejected": -0.5202240943908691,
+      "loss": 0.9021,
+      "nll_loss": 0.7989016771316528,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.0393654927611351,
+      "rewards/margins": 0.012656916864216328,
+      "rewards/rejected": -0.052022408694028854,
+      "step": 900
+    },
+    {
+      "epoch": 0.8516612072999532,
+      "grad_norm": 14.5,
+      "learning_rate": 3.2613709667171893e-08,
+      "log_odds_chosen": 0.22882740199565887,
+      "log_odds_ratio": -0.625227153301239,
+      "logits/chosen": -2.161898612976074,
+      "logits/rejected": -2.1723926067352295,
+      "logps/chosen": -0.39593061804771423,
+      "logps/rejected": -0.48067528009414673,
+      "loss": 0.9656,
+      "nll_loss": 0.8755291104316711,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.03959306702017784,
+      "rewards/margins": 0.008474460802972317,
+      "rewards/rejected": -0.048067521303892136,
+      "step": 910
+    },
+    {
+      "epoch": 0.8610201216658867,
+      "grad_norm": 16.75,
+      "learning_rate": 2.8694453138505296e-08,
+      "log_odds_chosen": 0.2396305501461029,
+      "log_odds_ratio": -0.6223834753036499,
+      "logits/chosen": -2.132664442062378,
+      "logits/rejected": -2.13667893409729,
+      "logps/chosen": -0.3883790373802185,
+      "logps/rejected": -0.4780302941799164,
+      "loss": 0.9318,
+      "nll_loss": 0.9020143747329712,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03883790224790573,
+      "rewards/margins": 0.008965130895376205,
+      "rewards/rejected": -0.04780303314328194,
+      "step": 920
+    },
+    {
+      "epoch": 0.8703790360318203,
+      "grad_norm": 15.625,
+      "learning_rate": 2.5011683239426847e-08,
+      "log_odds_chosen": 0.2851574420928955,
+      "log_odds_ratio": -0.5965331792831421,
+      "logits/chosen": -2.1351237297058105,
+      "logits/rejected": -2.108013153076172,
+      "logps/chosen": -0.4045374393463135,
+      "logps/rejected": -0.5147396922111511,
+      "loss": 1.0019,
+      "nll_loss": 0.9257608652114868,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.040453746914863586,
+      "rewards/margins": 0.011020222678780556,
+      "rewards/rejected": -0.05147396773099899,
+      "step": 930
+    },
+    {
+      "epoch": 0.8797379503977538,
+      "grad_norm": 15.125,
+      "learning_rate": 2.1569335369899884e-08,
+      "log_odds_chosen": 0.28318914771080017,
+      "log_odds_ratio": -0.5989499092102051,
+      "logits/chosen": -2.1903491020202637,
+      "logits/rejected": -2.1783673763275146,
+      "logps/chosen": -0.376907616853714,
+      "logps/rejected": -0.48921266198158264,
+      "loss": 0.9765,
+      "nll_loss": 0.953335165977478,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03769076615571976,
+      "rewards/margins": 0.011230507865548134,
+      "rewards/rejected": -0.04892126843333244,
+      "step": 940
+    },
+    {
+      "epoch": 0.8890968647636874,
+      "grad_norm": 12.875,
+      "learning_rate": 1.837108801542589e-08,
+      "log_odds_chosen": 0.3352145552635193,
+      "log_odds_ratio": -0.5945736169815063,
+      "logits/chosen": -2.170628786087036,
+      "logits/rejected": -2.1475720405578613,
+      "logps/chosen": -0.37348341941833496,
+      "logps/rejected": -0.5254369974136353,
+      "loss": 0.9728,
+      "nll_loss": 0.9157112240791321,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03734833747148514,
+      "rewards/margins": 0.01519535668194294,
+      "rewards/rejected": -0.052543699741363525,
+      "step": 950
+    },
+    {
+      "epoch": 0.8984557791296209,
+      "grad_norm": 16.125,
+      "learning_rate": 1.5420358816223e-08,
+      "log_odds_chosen": 0.2994464635848999,
+      "log_odds_ratio": -0.6000243425369263,
+      "logits/chosen": -2.1644070148468018,
+      "logits/rejected": -2.1604244709014893,
+      "logps/chosen": -0.4016776978969574,
+      "logps/rejected": -0.527344286441803,
+      "loss": 0.9684,
+      "nll_loss": 0.9093497395515442,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.04016777127981186,
+      "rewards/margins": 0.012566653080284595,
+      "rewards/rejected": -0.05273442342877388,
+      "step": 960
+    },
+    {
+      "epoch": 0.9078146934955545,
+      "grad_norm": 13.5,
+      "learning_rate": 1.2720300915142978e-08,
+      "log_odds_chosen": 0.25296837091445923,
+      "log_odds_ratio": -0.606758713722229,
+      "logits/chosen": -2.21331787109375,
+      "logits/rejected": -2.1903586387634277,
+      "logps/chosen": -0.3899661600589752,
+      "logps/rejected": -0.4821712076663971,
+      "loss": 1.0186,
+      "nll_loss": 0.8984857797622681,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.0389966145157814,
+      "rewards/margins": 0.009220501407980919,
+      "rewards/rejected": -0.04821711406111717,
+      "step": 970
+    },
+    {
+      "epoch": 0.917173607861488,
+      "grad_norm": 12.4375,
+      "learning_rate": 1.0273799588229659e-08,
+      "log_odds_chosen": 0.19899612665176392,
+      "log_odds_ratio": -0.6300643682479858,
+      "logits/chosen": -2.1826329231262207,
+      "logits/rejected": -2.171332836151123,
+      "logps/chosen": -0.3943658471107483,
+      "logps/rejected": -0.46531087160110474,
+      "loss": 1.0421,
+      "nll_loss": 0.9711662530899048,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03943658620119095,
+      "rewards/margins": 0.007094503380358219,
+      "rewards/rejected": -0.046531084924936295,
+      "step": 980
+    },
+    {
+      "epoch": 0.9265325222274217,
+      "grad_norm": 14.5,
+      "learning_rate": 8.08346916151903e-09,
+      "log_odds_chosen": 0.24496571719646454,
+      "log_odds_ratio": -0.6311475038528442,
+      "logits/chosen": -2.199125051498413,
+      "logits/rejected": -2.1684861183166504,
+      "logps/chosen": -0.399278849363327,
+      "logps/rejected": -0.5040075182914734,
+      "loss": 0.9278,
+      "nll_loss": 0.855401873588562,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0399278849363327,
+      "rewards/margins": 0.010472869500517845,
+      "rewards/rejected": -0.0504007525742054,
+      "step": 990
+    },
+    {
+      "epoch": 0.9358914365933552,
+      "grad_norm": 13.8125,
+      "learning_rate": 6.151650217376347e-09,
+      "log_odds_chosen": 0.26680201292037964,
+      "log_odds_ratio": -0.5990436673164368,
+      "logits/chosen": -2.1688525676727295,
+      "logits/rejected": -2.1509175300598145,
+      "logps/chosen": -0.39877527952194214,
+      "logps/rejected": -0.5044213533401489,
+      "loss": 0.9671,
+      "nll_loss": 0.9201105237007141,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.03987753018736839,
+      "rewards/margins": 0.01056460477411747,
+      "rewards/rejected": -0.05044213682413101,
+      "step": 1000
+    },
+    {
+      "epoch": 0.9452503509592888,
+      "grad_norm": 13.625,
+      "learning_rate": 4.480407093354566e-09,
+      "log_odds_chosen": 0.1911187767982483,
+      "log_odds_ratio": -0.6375032663345337,
+      "logits/chosen": -2.192441463470459,
+      "logits/rejected": -2.178623676300049,
+      "logps/chosen": -0.4046042561531067,
+      "logps/rejected": -0.46979087591171265,
+      "loss": 0.9988,
+      "nll_loss": 0.8718518018722534,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04046043008565903,
+      "rewards/margins": 0.006518671754747629,
+      "rewards/rejected": -0.04697910323739052,
+      "step": 1010
+    },
+    {
+      "epoch": 0.9546092653252223,
+      "grad_norm": 11.8125,
+      "learning_rate": 3.0715256762478825e-09,
+      "log_odds_chosen": 0.31750017404556274,
+      "log_odds_ratio": -0.5878022909164429,
+      "logits/chosen": -2.161133289337158,
+      "logits/rejected": -2.1433868408203125,
+      "logps/chosen": -0.3968185782432556,
+      "logps/rejected": -0.5128824710845947,
+      "loss": 0.9572,
+      "nll_loss": 0.9017167091369629,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.03968185931444168,
+      "rewards/margins": 0.01160639338195324,
+      "rewards/rejected": -0.05128825455904007,
+      "step": 1020
+    },
+    {
+      "epoch": 0.9639681796911558,
+      "grad_norm": 16.75,
+      "learning_rate": 1.926511493696936e-09,
+      "log_odds_chosen": 0.1701984405517578,
+      "log_odds_ratio": -0.6456406712532043,
+      "logits/chosen": -2.196162700653076,
+      "logits/rejected": -2.202619791030884,
+      "logps/chosen": -0.40211939811706543,
+      "logps/rejected": -0.4752349853515625,
+      "loss": 0.9981,
+      "nll_loss": 0.8766402006149292,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.040211938321590424,
+      "rewards/margins": 0.007311551831662655,
+      "rewards/rejected": -0.04752349108457565,
+      "step": 1030
+    },
+    {
+      "epoch": 0.9733270940570894,
+      "grad_norm": 14.75,
+      "learning_rate": 1.04658810538516e-09,
+      "log_odds_chosen": 0.23622456192970276,
+      "log_odds_ratio": -0.6107891798019409,
+      "logits/chosen": -2.175441265106201,
+      "logits/rejected": -2.1792852878570557,
+      "logps/chosen": -0.3729916214942932,
+      "logps/rejected": -0.45000284910202026,
+      "loss": 0.926,
+      "nll_loss": 0.8379205465316772,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.03729915991425514,
+      "rewards/margins": 0.0077011240646243095,
+      "rewards/rejected": -0.045000284910202026,
+      "step": 1040
+    },
+    {
+      "epoch": 0.9826860084230229,
+      "grad_norm": 15.0625,
+      "learning_rate": 4.3269579554558455e-10,
+      "log_odds_chosen": 0.19562143087387085,
+      "log_odds_ratio": -0.6367918848991394,
+      "logits/chosen": -2.2001359462738037,
+      "logits/rejected": -2.183256149291992,
+      "logps/chosen": -0.4047182500362396,
+      "logps/rejected": -0.48164814710617065,
+      "loss": 0.9974,
+      "nll_loss": 1.0015695095062256,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04047182947397232,
+      "rewards/margins": 0.007692983839660883,
+      "rewards/rejected": -0.04816481098532677,
+      "step": 1050
+    },
+    {
+      "epoch": 0.9920449227889565,
+      "grad_norm": 14.4375,
+      "learning_rate": 8.549056817513944e-11,
+      "log_odds_chosen": 0.29663318395614624,
+      "log_odds_ratio": -0.6037989854812622,
+      "logits/chosen": -2.197646379470825,
+      "logits/rejected": -2.1741132736206055,
+      "logps/chosen": -0.3990115523338318,
+      "logps/rejected": -0.5252410173416138,
+      "loss": 0.9644,
+      "nll_loss": 0.9385454058647156,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03990115597844124,
+      "rewards/margins": 0.012622940354049206,
+      "rewards/rejected": -0.05252409726381302,
+      "step": 1060
+    },
+    {
+      "epoch": 0.9995320542817033,
+      "step": 1068,
+      "total_flos": 0.0,
+      "train_loss": 1.0056083492571941,
+      "train_runtime": 16376.6673,
+      "train_samples_per_second": 2.088,
+      "train_steps_per_second": 0.065
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1068,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}