Model save

README.md

@@ -0,0 +1,74 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-reference
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-reference
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0429
+- Rewards/chosen: -0.5987
+- Rewards/rejected: -10.2552
+- Rewards/accuracies: 0.9741
+- Rewards/margins: 9.6565
+- Logps/rejected: -175.1052
+- Logps/chosen: -304.8906
+- Logits/rejected: -1.9643
+- Logits/chosen: -2.1592
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.0811        | 0.29  | 500  | 0.0543          | -0.5167        | -8.9128          | 0.9720             | 8.3961          | -161.6813      | -304.0705    | -2.0037         | -2.1857       |
+| 0.0362        | 0.57  | 1000 | 0.0483          | -0.4980        | -9.5824          | 0.9720             | 9.0844          | -168.3771      | -303.8834    | -2.0113         | -2.2030       |
+| 0.0318        | 0.86  | 1500 | 0.0442          | -0.8458        | -10.5987         | 0.9720             | 9.7529          | -178.5403      | -307.3617    | -1.9506         | -2.1461       |
+
+
+### Framework versions
+
+- Transformers 4.36.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.159156560897827,
+    "eval_logits/rejected": -1.9642761945724487,
+    "eval_logps/chosen": -304.89056396484375,
+    "eval_logps/rejected": -175.10516357421875,
+    "eval_loss": 0.04293975234031677,
+    "eval_rewards/accuracies": 0.9741379022598267,
+    "eval_rewards/chosen": -0.5987315773963928,
+    "eval_rewards/margins": 9.656463623046875,
+    "eval_rewards/rejected": -10.255194664001465,
+    "eval_runtime": 497.484,
+    "eval_samples": 1843,
+    "eval_samples_per_second": 3.705,
+    "eval_steps_per_second": 0.117,
+    "train_loss": 0.06308119606848633,
+    "train_runtime": 23531.0319,
+    "train_samples": 55762,
+    "train_samples_per_second": 2.37,
+    "train_steps_per_second": 0.074
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.159156560897827,
+    "eval_logits/rejected": -1.9642761945724487,
+    "eval_logps/chosen": -304.89056396484375,
+    "eval_logps/rejected": -175.10516357421875,
+    "eval_loss": 0.04293975234031677,
+    "eval_rewards/accuracies": 0.9741379022598267,
+    "eval_rewards/chosen": -0.5987315773963928,
+    "eval_rewards/margins": 9.656463623046875,
+    "eval_rewards/rejected": -10.255194664001465,
+    "eval_runtime": 497.484,
+    "eval_samples": 1843,
+    "eval_samples_per_second": 3.705,
+    "eval_steps_per_second": 0.117
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905838749b852fa9332ddfae123aa13b6b7c0349bd967c2a35ee7dc50e9ea1dc
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9fe8a51c40d49c2089461ed6bcf0ffcc80af50863a6acb9b33406f500ad379c
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ecfac24d65a7bbf384ff46d62368f2d99a8fef93ff5aa780daae30a41d9a75a
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Feb01_13-51-56_node01/events.out.tfevents.1706766942.node01.3302100.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58ce9b9f34b5cbd0918834ba4b73c33f7b78b8f12733296a6fc6144cdcb6e622
+size 117846

runs/Feb01_13-51-56_node01/events.out.tfevents.1706790971.node01.3302100.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:464dbb90fc1872bd1fdb9d0efd96060d8e99d51e4aab550ad26cd2dbef2d0392
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.06308119606848633,
+    "train_runtime": 23531.0319,
+    "train_samples": 55762,
+    "train_samples_per_second": 2.37,
+    "train_steps_per_second": 0.074
+}

trainer_state.json

@@ -0,0 +1,2528 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1743,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.857142857142857e-09,
+      "logits/chosen": -2.8170976638793945,
+      "logits/rejected": -2.583425998687744,
+      "logps/chosen": -267.58819580078125,
+      "logps/rejected": -55.24143981933594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.857142857142857e-08,
+      "logits/chosen": -2.728940486907959,
+      "logits/rejected": -2.597886800765991,
+      "logps/chosen": -287.7469482421875,
+      "logps/rejected": -66.66583251953125,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": -0.003922327421605587,
+      "rewards/margins": -0.0009250044822692871,
+      "rewards/rejected": -0.0029973229393363,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.714285714285714e-08,
+      "logits/chosen": -2.6918883323669434,
+      "logits/rejected": -2.6110966205596924,
+      "logps/chosen": -280.5570068359375,
+      "logps/rejected": -69.74317169189453,
+      "loss": 0.6445,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.10821928083896637,
+      "rewards/margins": 0.11207082122564316,
+      "rewards/rejected": -0.003851559478789568,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.571428571428572e-08,
+      "logits/chosen": -2.6756694316864014,
+      "logits/rejected": -2.5497803688049316,
+      "logps/chosen": -334.42138671875,
+      "logps/rejected": -74.35704040527344,
+      "loss": 0.5256,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.3521518111228943,
+      "rewards/margins": 0.4118792414665222,
+      "rewards/rejected": -0.05972742289304733,
+      "step": 30
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1428571428571427e-07,
+      "logits/chosen": -2.585674524307251,
+      "logits/rejected": -2.4876582622528076,
+      "logps/chosen": -259.1927185058594,
+      "logps/rejected": -70.0163345336914,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.5247544050216675,
+      "rewards/margins": 0.7782431244850159,
+      "rewards/rejected": -0.2534886598587036,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4285714285714285e-07,
+      "logits/chosen": -2.556175947189331,
+      "logits/rejected": -2.4422554969787598,
+      "logps/chosen": -293.9497375488281,
+      "logps/rejected": -78.49222564697266,
+      "loss": 0.3084,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.4779079556465149,
+      "rewards/margins": 1.3577072620391846,
+      "rewards/rejected": -0.8797992467880249,
+      "step": 50
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.7142857142857143e-07,
+      "logits/chosen": -2.5218682289123535,
+      "logits/rejected": -2.413239002227783,
+      "logps/chosen": -348.8528747558594,
+      "logps/rejected": -90.35295867919922,
+      "loss": 0.2281,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.8038774728775024,
+      "rewards/margins": 2.1713979244232178,
+      "rewards/rejected": -1.3675200939178467,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2e-07,
+      "logits/chosen": -2.5286450386047363,
+      "logits/rejected": -2.4316279888153076,
+      "logps/chosen": -277.53424072265625,
+      "logps/rejected": -77.01128387451172,
+      "loss": 0.2105,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.6769354343414307,
+      "rewards/margins": 2.166170358657837,
+      "rewards/rejected": -1.4892350435256958,
+      "step": 70
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.2857142857142855e-07,
+      "logits/chosen": -2.498818874359131,
+      "logits/rejected": -2.377124071121216,
+      "logps/chosen": -307.00531005859375,
+      "logps/rejected": -99.22511291503906,
+      "loss": 0.1428,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.6349704265594482,
+      "rewards/margins": 2.8562676906585693,
+      "rewards/rejected": -2.2212975025177,
+      "step": 80
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.571428571428571e-07,
+      "logits/chosen": -2.5420005321502686,
+      "logits/rejected": -2.4419803619384766,
+      "logps/chosen": -297.6865539550781,
+      "logps/rejected": -105.5224609375,
+      "loss": 0.1443,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.5000184774398804,
+      "rewards/margins": 3.1510608196258545,
+      "rewards/rejected": -2.6510424613952637,
+      "step": 90
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.857142857142857e-07,
+      "logits/chosen": -2.4766311645507812,
+      "logits/rejected": -2.364316701889038,
+      "logps/chosen": -269.93719482421875,
+      "logps/rejected": -100.0908432006836,
+      "loss": 0.1393,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.4442061483860016,
+      "rewards/margins": 3.083071231842041,
+      "rewards/rejected": -2.638864517211914,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.142857142857143e-07,
+      "logits/chosen": -2.388690233230591,
+      "logits/rejected": -2.3019070625305176,
+      "logps/chosen": -259.3259582519531,
+      "logps/rejected": -92.60478210449219,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.3840440511703491,
+      "rewards/margins": 3.357806444168091,
+      "rewards/rejected": -2.973762035369873,
+      "step": 110
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4285714285714286e-07,
+      "logits/chosen": -2.413675546646118,
+      "logits/rejected": -2.366042375564575,
+      "logps/chosen": -291.9371643066406,
+      "logps/rejected": -122.68412017822266,
+      "loss": 0.1103,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.07346852123737335,
+      "rewards/margins": 3.9470927715301514,
+      "rewards/rejected": -3.873624324798584,
+      "step": 120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.7142857142857145e-07,
+      "logits/chosen": -2.2968249320983887,
+      "logits/rejected": -2.225172519683838,
+      "logps/chosen": -244.48300170898438,
+      "logps/rejected": -106.67143249511719,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.45761212706565857,
+      "rewards/margins": 3.7605834007263184,
+      "rewards/rejected": -4.21819543838501,
+      "step": 130
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4e-07,
+      "logits/chosen": -2.4523983001708984,
+      "logits/rejected": -2.301741123199463,
+      "logps/chosen": -299.51251220703125,
+      "logps/rejected": -111.51058197021484,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.3345158100128174,
+      "rewards/margins": 4.5269455909729,
+      "rewards/rejected": -4.192429542541504,
+      "step": 140
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.285714285714285e-07,
+      "logits/chosen": -2.2728230953216553,
+      "logits/rejected": -2.1703383922576904,
+      "logps/chosen": -217.3053741455078,
+      "logps/rejected": -100.21586608886719,
+      "loss": 0.0937,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.14594963192939758,
+      "rewards/margins": 3.915192127227783,
+      "rewards/rejected": -3.76924204826355,
+      "step": 150
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.571428571428571e-07,
+      "logits/chosen": -2.311068058013916,
+      "logits/rejected": -2.1951611042022705,
+      "logps/chosen": -224.5479736328125,
+      "logps/rejected": -97.22464752197266,
+      "loss": 0.081,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.23994378745555878,
+      "rewards/margins": 4.240386009216309,
+      "rewards/rejected": -4.000441551208496,
+      "step": 160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.857142857142857e-07,
+      "logits/chosen": -2.4215264320373535,
+      "logits/rejected": -2.3117451667785645,
+      "logps/chosen": -338.24591064453125,
+      "logps/rejected": -135.39236450195312,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.741433322429657,
+      "rewards/margins": 6.07324743270874,
+      "rewards/rejected": -5.331814765930176,
+      "step": 170
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.984056122448979e-07,
+      "logits/chosen": -2.376217842102051,
+      "logits/rejected": -2.2611336708068848,
+      "logps/chosen": -308.8148498535156,
+      "logps/rejected": -132.30233764648438,
+      "loss": 0.1005,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.16114525496959686,
+      "rewards/margins": 5.889215469360352,
+      "rewards/rejected": -5.72806978225708,
+      "step": 180
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.952168367346938e-07,
+      "logits/chosen": -2.3446619510650635,
+      "logits/rejected": -2.239445209503174,
+      "logps/chosen": -316.2629699707031,
+      "logps/rejected": -133.4289093017578,
+      "loss": 0.0733,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.2028081864118576,
+      "rewards/margins": 5.608849048614502,
+      "rewards/rejected": -5.811657905578613,
+      "step": 190
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.920280612244897e-07,
+      "logits/chosen": -2.263739585876465,
+      "logits/rejected": -2.1694753170013428,
+      "logps/chosen": -233.5961456298828,
+      "logps/rejected": -109.9480209350586,
+      "loss": 0.0772,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.2176302969455719,
+      "rewards/margins": 5.1773786544799805,
+      "rewards/rejected": -4.959748268127441,
+      "step": 200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.888392857142857e-07,
+      "logits/chosen": -2.3166961669921875,
+      "logits/rejected": -2.2644801139831543,
+      "logps/chosen": -266.83099365234375,
+      "logps/rejected": -127.63531494140625,
+      "loss": 0.0682,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.30769938230514526,
+      "rewards/margins": 5.930529594421387,
+      "rewards/rejected": -5.622830390930176,
+      "step": 210
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.856505102040816e-07,
+      "logits/chosen": -2.289324998855591,
+      "logits/rejected": -2.145235538482666,
+      "logps/chosen": -287.94903564453125,
+      "logps/rejected": -135.3939666748047,
+      "loss": 0.0824,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.18774142861366272,
+      "rewards/margins": 6.2231669425964355,
+      "rewards/rejected": -6.035426139831543,
+      "step": 220
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.824617346938776e-07,
+      "logits/chosen": -2.216264009475708,
+      "logits/rejected": -2.047607183456421,
+      "logps/chosen": -271.09869384765625,
+      "logps/rejected": -129.23095703125,
+      "loss": 0.1054,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.05019025877118111,
+      "rewards/margins": 6.176311016082764,
+      "rewards/rejected": -6.126120567321777,
+      "step": 230
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.792729591836734e-07,
+      "logits/chosen": -2.2174153327941895,
+      "logits/rejected": -2.094043254852295,
+      "logps/chosen": -295.2961120605469,
+      "logps/rejected": -132.24790954589844,
+      "loss": 0.0579,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.24180030822753906,
+      "rewards/margins": 6.183707237243652,
+      "rewards/rejected": -5.941906929016113,
+      "step": 240
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.760841836734694e-07,
+      "logits/chosen": -2.3349456787109375,
+      "logits/rejected": -2.2391867637634277,
+      "logps/chosen": -301.3727722167969,
+      "logps/rejected": -140.08639526367188,
+      "loss": 0.0578,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2325439751148224,
+      "rewards/margins": 6.470588684082031,
+      "rewards/rejected": -6.703132629394531,
+      "step": 250
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.728954081632653e-07,
+      "logits/chosen": -2.2977206707000732,
+      "logits/rejected": -2.1363391876220703,
+      "logps/chosen": -305.45477294921875,
+      "logps/rejected": -140.94717407226562,
+      "loss": 0.0599,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.0075416178442537785,
+      "rewards/margins": 6.882362365722656,
+      "rewards/rejected": -6.8899030685424805,
+      "step": 260
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.697066326530612e-07,
+      "logits/chosen": -2.2209079265594482,
+      "logits/rejected": -2.1426846981048584,
+      "logps/chosen": -269.04010009765625,
+      "logps/rejected": -142.6284942626953,
+      "loss": 0.056,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.23809853196144104,
+      "rewards/margins": 6.732804298400879,
+      "rewards/rejected": -6.970902919769287,
+      "step": 270
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.665178571428571e-07,
+      "logits/chosen": -2.3430752754211426,
+      "logits/rejected": -2.2075252532958984,
+      "logps/chosen": -314.6163024902344,
+      "logps/rejected": -146.22244262695312,
+      "loss": 0.0529,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08103661239147186,
+      "rewards/margins": 7.245053768157959,
+      "rewards/rejected": -7.3260908126831055,
+      "step": 280
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.6332908163265305e-07,
+      "logits/chosen": -2.2507033348083496,
+      "logits/rejected": -2.094573736190796,
+      "logps/chosen": -262.8319396972656,
+      "logps/rejected": -141.3706817626953,
+      "loss": 0.0589,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.06328751146793365,
+      "rewards/margins": 6.6208062171936035,
+      "rewards/rejected": -6.684094429016113,
+      "step": 290
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.60140306122449e-07,
+      "logits/chosen": -2.2667717933654785,
+      "logits/rejected": -2.16914701461792,
+      "logps/chosen": -282.3851623535156,
+      "logps/rejected": -144.63890075683594,
+      "loss": 0.0585,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.35556235909461975,
+      "rewards/margins": 6.836700439453125,
+      "rewards/rejected": -7.192262172698975,
+      "step": 300
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.569515306122449e-07,
+      "logits/chosen": -2.2799079418182373,
+      "logits/rejected": -2.1546857357025146,
+      "logps/chosen": -307.00457763671875,
+      "logps/rejected": -144.7294921875,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.4573780596256256,
+      "rewards/margins": 7.150294303894043,
+      "rewards/rejected": -6.6929168701171875,
+      "step": 310
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.5376275510204084e-07,
+      "logits/chosen": -2.2312560081481934,
+      "logits/rejected": -2.0914812088012695,
+      "logps/chosen": -261.8872985839844,
+      "logps/rejected": -139.28701782226562,
+      "loss": 0.0619,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.15933240950107574,
+      "rewards/margins": 6.717618465423584,
+      "rewards/rejected": -6.876950740814209,
+      "step": 320
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.505739795918367e-07,
+      "logits/chosen": -2.2225937843322754,
+      "logits/rejected": -2.0411500930786133,
+      "logps/chosen": -304.5137634277344,
+      "logps/rejected": -132.83473205566406,
+      "loss": 0.065,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.07894410938024521,
+      "rewards/margins": 6.419223785400391,
+      "rewards/rejected": -6.340278625488281,
+      "step": 330
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.473852040816326e-07,
+      "logits/chosen": -2.304166793823242,
+      "logits/rejected": -2.144609212875366,
+      "logps/chosen": -274.3529968261719,
+      "logps/rejected": -131.06246948242188,
+      "loss": 0.0907,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.192825049161911,
+      "rewards/margins": 6.38867712020874,
+      "rewards/rejected": -6.5815019607543945,
+      "step": 340
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.441964285714285e-07,
+      "logits/chosen": -2.305800676345825,
+      "logits/rejected": -2.111426830291748,
+      "logps/chosen": -340.38702392578125,
+      "logps/rejected": -148.1328887939453,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.030660927295684814,
+      "rewards/margins": 7.517220973968506,
+      "rewards/rejected": -7.547882080078125,
+      "step": 350
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.4100765306122445e-07,
+      "logits/chosen": -2.249474287033081,
+      "logits/rejected": -2.114851474761963,
+      "logps/chosen": -276.80303955078125,
+      "logps/rejected": -139.46585083007812,
+      "loss": 0.06,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.2887513041496277,
+      "rewards/margins": 7.209240913391113,
+      "rewards/rejected": -7.497992038726807,
+      "step": 360
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.378188775510204e-07,
+      "logits/chosen": -2.227515935897827,
+      "logits/rejected": -2.0700523853302,
+      "logps/chosen": -279.75494384765625,
+      "logps/rejected": -127.5013198852539,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.12594351172447205,
+      "rewards/margins": 6.841870307922363,
+      "rewards/rejected": -6.715925693511963,
+      "step": 370
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.346301020408163e-07,
+      "logits/chosen": -2.196239948272705,
+      "logits/rejected": -2.0718905925750732,
+      "logps/chosen": -291.7792663574219,
+      "logps/rejected": -154.97567749023438,
+      "loss": 0.0363,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.4449889063835144,
+      "rewards/margins": 6.8718414306640625,
+      "rewards/rejected": -7.316830635070801,
+      "step": 380
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.314413265306122e-07,
+      "logits/chosen": -2.2047321796417236,
+      "logits/rejected": -2.04673171043396,
+      "logps/chosen": -278.8887023925781,
+      "logps/rejected": -148.29612731933594,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.8236015439033508,
+      "rewards/margins": 8.402433395385742,
+      "rewards/rejected": -7.578830718994141,
+      "step": 390
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.282525510204081e-07,
+      "logits/chosen": -2.2842113971710205,
+      "logits/rejected": -2.1249568462371826,
+      "logps/chosen": -252.1570587158203,
+      "logps/rejected": -131.91351318359375,
+      "loss": 0.0658,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.4603995382785797,
+      "rewards/margins": 6.588088035583496,
+      "rewards/rejected": -7.048487186431885,
+      "step": 400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.2506377551020405e-07,
+      "logits/chosen": -2.2204508781433105,
+      "logits/rejected": -2.0328991413116455,
+      "logps/chosen": -277.8280334472656,
+      "logps/rejected": -131.99388122558594,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.6896305084228516,
+      "rewards/margins": 6.525734901428223,
+      "rewards/rejected": -7.215365409851074,
+      "step": 410
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.21875e-07,
+      "logits/chosen": -2.281752586364746,
+      "logits/rejected": -2.1253433227539062,
+      "logps/chosen": -295.5191345214844,
+      "logps/rejected": -153.25,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.4880848824977875,
+      "rewards/margins": 7.4977922439575195,
+      "rewards/rejected": -7.98587703704834,
+      "step": 420
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.186862244897959e-07,
+      "logits/chosen": -2.2700979709625244,
+      "logits/rejected": -2.0792222023010254,
+      "logps/chosen": -246.54293823242188,
+      "logps/rejected": -140.80882263183594,
+      "loss": 0.047,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.25689664483070374,
+      "rewards/margins": 7.780982971191406,
+      "rewards/rejected": -8.037880897521973,
+      "step": 430
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.1549744897959185e-07,
+      "logits/chosen": -2.2732179164886475,
+      "logits/rejected": -2.10884690284729,
+      "logps/chosen": -301.9032897949219,
+      "logps/rejected": -160.38320922851562,
+      "loss": 0.0668,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.4491173326969147,
+      "rewards/margins": 8.146552085876465,
+      "rewards/rejected": -8.595669746398926,
+      "step": 440
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.123086734693877e-07,
+      "logits/chosen": -2.259704113006592,
+      "logits/rejected": -2.1263084411621094,
+      "logps/chosen": -248.52328491210938,
+      "logps/rejected": -142.90264892578125,
+      "loss": 0.037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.04825396090745926,
+      "rewards/margins": 7.279501914978027,
+      "rewards/rejected": -7.327755928039551,
+      "step": 450
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.0911989795918365e-07,
+      "logits/chosen": -2.1421685218811035,
+      "logits/rejected": -2.0395822525024414,
+      "logps/chosen": -253.42160034179688,
+      "logps/rejected": -134.23333740234375,
+      "loss": 0.0699,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.702133059501648,
+      "rewards/margins": 6.782039642333984,
+      "rewards/rejected": -7.484173774719238,
+      "step": 460
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.059311224489796e-07,
+      "logits/chosen": -2.2776074409484863,
+      "logits/rejected": -2.075753688812256,
+      "logps/chosen": -261.2068176269531,
+      "logps/rejected": -140.44773864746094,
+      "loss": 0.0891,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.07892769575119019,
+      "rewards/margins": 7.501675605773926,
+      "rewards/rejected": -7.580602169036865,
+      "step": 470
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.027423469387755e-07,
+      "logits/chosen": -2.144608974456787,
+      "logits/rejected": -2.0202698707580566,
+      "logps/chosen": -230.21023559570312,
+      "logps/rejected": -146.96124267578125,
+      "loss": 0.1102,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.1200422048568726,
+      "rewards/margins": 7.595203399658203,
+      "rewards/rejected": -8.715245246887207,
+      "step": 480
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 3.9955357142857144e-07,
+      "logits/chosen": -2.1214406490325928,
+      "logits/rejected": -1.9482629299163818,
+      "logps/chosen": -271.87554931640625,
+      "logps/rejected": -156.08773803710938,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.0995118618011475,
+      "rewards/margins": 7.6725311279296875,
+      "rewards/rejected": -8.772043228149414,
+      "step": 490
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.963647959183674e-07,
+      "logits/chosen": -2.222761392593384,
+      "logits/rejected": -2.065107822418213,
+      "logps/chosen": -331.9025573730469,
+      "logps/rejected": -174.4883575439453,
+      "loss": 0.0811,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.9387734532356262,
+      "rewards/margins": 8.301498413085938,
+      "rewards/rejected": -9.240272521972656,
+      "step": 500
+    },
+    {
+      "epoch": 0.29,
+      "eval_logits/chosen": -2.1856682300567627,
+      "eval_logits/rejected": -2.003657579421997,
+      "eval_logps/chosen": -304.0705261230469,
+      "eval_logps/rejected": -161.6812744140625,
+      "eval_loss": 0.05427403748035431,
+      "eval_rewards/accuracies": 0.9719827771186829,
+      "eval_rewards/chosen": -0.516727089881897,
+      "eval_rewards/margins": 8.396077156066895,
+      "eval_rewards/rejected": -8.91280460357666,
+      "eval_runtime": 515.5452,
+      "eval_samples_per_second": 3.575,
+      "eval_steps_per_second": 0.113,
+      "step": 500
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.931760204081632e-07,
+      "logits/chosen": -2.1678292751312256,
+      "logits/rejected": -2.0178639888763428,
+      "logps/chosen": -281.6822509765625,
+      "logps/rejected": -152.28237915039062,
+      "loss": 0.0387,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07496733963489532,
+      "rewards/margins": 8.705911636352539,
+      "rewards/rejected": -8.63094425201416,
+      "step": 510
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.8998724489795913e-07,
+      "logits/chosen": -2.124497652053833,
+      "logits/rejected": -1.9633057117462158,
+      "logps/chosen": -272.04278564453125,
+      "logps/rejected": -154.85665893554688,
+      "loss": 0.0747,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.6167508363723755,
+      "rewards/margins": 7.811673164367676,
+      "rewards/rejected": -8.428422927856445,
+      "step": 520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.8679846938775506e-07,
+      "logits/chosen": -2.0578837394714355,
+      "logits/rejected": -1.9298282861709595,
+      "logps/chosen": -290.99639892578125,
+      "logps/rejected": -168.87124633789062,
+      "loss": 0.0378,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.20361948013305664,
+      "rewards/margins": 9.225129127502441,
+      "rewards/rejected": -9.428749084472656,
+      "step": 530
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.83609693877551e-07,
+      "logits/chosen": -2.1337828636169434,
+      "logits/rejected": -1.9600626230239868,
+      "logps/chosen": -297.354248046875,
+      "logps/rejected": -156.31643676757812,
+      "loss": 0.0215,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2819109261035919,
+      "rewards/margins": 8.338506698608398,
+      "rewards/rejected": -8.056594848632812,
+      "step": 540
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.804209183673469e-07,
+      "logits/chosen": -2.1427624225616455,
+      "logits/rejected": -1.9869788885116577,
+      "logps/chosen": -314.75238037109375,
+      "logps/rejected": -156.91665649414062,
+      "loss": 0.0719,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07266904413700104,
+      "rewards/margins": 8.285478591918945,
+      "rewards/rejected": -8.358146667480469,
+      "step": 550
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.7723214285714285e-07,
+      "logits/chosen": -2.193801164627075,
+      "logits/rejected": -2.023911476135254,
+      "logps/chosen": -280.3459777832031,
+      "logps/rejected": -159.136474609375,
+      "loss": 0.043,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6412056684494019,
+      "rewards/margins": 8.589024543762207,
+      "rewards/rejected": -9.230230331420898,
+      "step": 560
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.7404336734693873e-07,
+      "logits/chosen": -2.1136326789855957,
+      "logits/rejected": -1.970041275024414,
+      "logps/chosen": -288.44378662109375,
+      "logps/rejected": -163.71575927734375,
+      "loss": 0.0628,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.7930206656455994,
+      "rewards/margins": 8.54659652709961,
+      "rewards/rejected": -9.339616775512695,
+      "step": 570
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.7085459183673466e-07,
+      "logits/chosen": -2.181980848312378,
+      "logits/rejected": -2.0017356872558594,
+      "logps/chosen": -269.84637451171875,
+      "logps/rejected": -161.8075714111328,
+      "loss": 0.0811,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.1092716455459595,
+      "rewards/margins": 8.634279251098633,
+      "rewards/rejected": -9.743551254272461,
+      "step": 580
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.676658163265306e-07,
+      "logits/chosen": -2.1273303031921387,
+      "logits/rejected": -1.9688310623168945,
+      "logps/chosen": -321.7066650390625,
+      "logps/rejected": -174.16650390625,
+      "loss": 0.0425,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19933632016181946,
+      "rewards/margins": 9.354225158691406,
+      "rewards/rejected": -9.553561210632324,
+      "step": 590
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.644770408163265e-07,
+      "logits/chosen": -2.167726993560791,
+      "logits/rejected": -1.9377820491790771,
+      "logps/chosen": -290.7398376464844,
+      "logps/rejected": -156.63296508789062,
+      "loss": 0.0404,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.34938400983810425,
+      "rewards/margins": 8.953282356262207,
+      "rewards/rejected": -9.302667617797852,
+      "step": 600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.6128826530612245e-07,
+      "logits/chosen": -2.037559747695923,
+      "logits/rejected": -1.890639305114746,
+      "logps/chosen": -251.7969970703125,
+      "logps/rejected": -145.12217712402344,
+      "loss": 0.0411,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.910443902015686,
+      "rewards/margins": 7.705912113189697,
+      "rewards/rejected": -8.61635684967041,
+      "step": 610
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.580994897959184e-07,
+      "logits/chosen": -2.1508290767669678,
+      "logits/rejected": -1.962224006652832,
+      "logps/chosen": -297.48358154296875,
+      "logps/rejected": -163.69259643554688,
+      "loss": 0.0546,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33932703733444214,
+      "rewards/margins": 9.108827590942383,
+      "rewards/rejected": -9.448153495788574,
+      "step": 620
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5491071428571426e-07,
+      "logits/chosen": -2.179138422012329,
+      "logits/rejected": -2.000100612640381,
+      "logps/chosen": -311.79119873046875,
+      "logps/rejected": -169.6217041015625,
+      "loss": 0.044,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.6057596206665039,
+      "rewards/margins": 8.855595588684082,
+      "rewards/rejected": -9.461355209350586,
+      "step": 630
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.517219387755102e-07,
+      "logits/chosen": -2.2315735816955566,
+      "logits/rejected": -2.1057209968566895,
+      "logps/chosen": -287.6869812011719,
+      "logps/rejected": -158.8007354736328,
+      "loss": 0.0373,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.49593955278396606,
+      "rewards/margins": 7.963850975036621,
+      "rewards/rejected": -8.459790229797363,
+      "step": 640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.485331632653061e-07,
+      "logits/chosen": -2.150575876235962,
+      "logits/rejected": -2.0577378273010254,
+      "logps/chosen": -261.49163818359375,
+      "logps/rejected": -165.69863891601562,
+      "loss": 0.0429,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.6350365281105042,
+      "rewards/margins": 8.936600685119629,
+      "rewards/rejected": -9.571638107299805,
+      "step": 650
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.4534438775510205e-07,
+      "logits/chosen": -2.080155849456787,
+      "logits/rejected": -1.9665069580078125,
+      "logps/chosen": -288.9784240722656,
+      "logps/rejected": -173.96405029296875,
+      "loss": 0.0592,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.8485971689224243,
+      "rewards/margins": 9.569430351257324,
+      "rewards/rejected": -10.418027877807617,
+      "step": 660
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.4215561224489793e-07,
+      "logits/chosen": -2.1020286083221436,
+      "logits/rejected": -2.015242338180542,
+      "logps/chosen": -286.51593017578125,
+      "logps/rejected": -197.00831604003906,
+      "loss": 0.0479,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8589476346969604,
+      "rewards/margins": 11.049829483032227,
+      "rewards/rejected": -11.90877628326416,
+      "step": 670
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.3896683673469386e-07,
+      "logits/chosen": -2.207352876663208,
+      "logits/rejected": -2.026625394821167,
+      "logps/chosen": -283.9382019042969,
+      "logps/rejected": -170.56997680664062,
+      "loss": 0.0686,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.428898811340332,
+      "rewards/margins": 8.341761589050293,
+      "rewards/rejected": -9.770662307739258,
+      "step": 680
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3577806122448974e-07,
+      "logits/chosen": -2.226600408554077,
+      "logits/rejected": -1.9801476001739502,
+      "logps/chosen": -318.254150390625,
+      "logps/rejected": -169.10043334960938,
+      "loss": 0.0859,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8905173540115356,
+      "rewards/margins": 8.872238159179688,
+      "rewards/rejected": -9.762755393981934,
+      "step": 690
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3258928571428567e-07,
+      "logits/chosen": -2.1024184226989746,
+      "logits/rejected": -1.9208921194076538,
+      "logps/chosen": -336.98724365234375,
+      "logps/rejected": -168.90625,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.09593725204467773,
+      "rewards/margins": 10.029386520385742,
+      "rewards/rejected": -10.125323295593262,
+      "step": 700
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.294005102040816e-07,
+      "logits/chosen": -2.137101650238037,
+      "logits/rejected": -1.9699983596801758,
+      "logps/chosen": -261.7487487792969,
+      "logps/rejected": -156.60157775878906,
+      "loss": 0.0614,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6025804281234741,
+      "rewards/margins": 8.209660530090332,
+      "rewards/rejected": -8.812241554260254,
+      "step": 710
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.2621173469387753e-07,
+      "logits/chosen": -2.2646114826202393,
+      "logits/rejected": -2.1247591972351074,
+      "logps/chosen": -329.1217041015625,
+      "logps/rejected": -185.3964385986328,
+      "loss": 0.0578,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.2518744468688965,
+      "rewards/margins": 9.136481285095215,
+      "rewards/rejected": -10.388354301452637,
+      "step": 720
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2302295918367346e-07,
+      "logits/chosen": -2.3025715351104736,
+      "logits/rejected": -2.1256372928619385,
+      "logps/chosen": -310.5058288574219,
+      "logps/rejected": -170.10592651367188,
+      "loss": 0.029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1804181337356567,
+      "rewards/margins": 8.121767044067383,
+      "rewards/rejected": -9.302184104919434,
+      "step": 730
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.198341836734694e-07,
+      "logits/chosen": -2.2341511249542236,
+      "logits/rejected": -2.04396915435791,
+      "logps/chosen": -328.27667236328125,
+      "logps/rejected": -192.09335327148438,
+      "loss": 0.0565,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.02377413585782051,
+      "rewards/margins": 11.305741310119629,
+      "rewards/rejected": -11.32951545715332,
+      "step": 740
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.1664540816326527e-07,
+      "logits/chosen": -2.1826584339141846,
+      "logits/rejected": -2.023871660232544,
+      "logps/chosen": -276.5951843261719,
+      "logps/rejected": -160.23538208007812,
+      "loss": 0.0376,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.3585100769996643,
+      "rewards/margins": 9.415512084960938,
+      "rewards/rejected": -9.77402114868164,
+      "step": 750
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.134566326530612e-07,
+      "logits/chosen": -2.227428436279297,
+      "logits/rejected": -2.070985794067383,
+      "logps/chosen": -294.4335021972656,
+      "logps/rejected": -169.70211791992188,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.31169813871383667,
+      "rewards/margins": 9.067781448364258,
+      "rewards/rejected": -9.379480361938477,
+      "step": 760
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.1026785714285713e-07,
+      "logits/chosen": -2.1913247108459473,
+      "logits/rejected": -1.9936134815216064,
+      "logps/chosen": -328.67083740234375,
+      "logps/rejected": -175.294921875,
+      "loss": 0.0432,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.3559989333152771,
+      "rewards/margins": 9.043487548828125,
+      "rewards/rejected": -9.399484634399414,
+      "step": 770
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.0707908163265306e-07,
+      "logits/chosen": -2.1028852462768555,
+      "logits/rejected": -1.8983999490737915,
+      "logps/chosen": -273.2193908691406,
+      "logps/rejected": -158.0215301513672,
+      "loss": 0.0428,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.963762104511261,
+      "rewards/margins": 8.092933654785156,
+      "rewards/rejected": -9.056695938110352,
+      "step": 780
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.03890306122449e-07,
+      "logits/chosen": -2.1897473335266113,
+      "logits/rejected": -1.9860061407089233,
+      "logps/chosen": -317.2612609863281,
+      "logps/rejected": -167.63690185546875,
+      "loss": 0.0308,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.3040831685066223,
+      "rewards/margins": 8.782797813415527,
+      "rewards/rejected": -9.086880683898926,
+      "step": 790
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.007015306122449e-07,
+      "logits/chosen": -2.2554848194122314,
+      "logits/rejected": -2.0466835498809814,
+      "logps/chosen": -359.5973815917969,
+      "logps/rejected": -168.97488403320312,
+      "loss": 0.0485,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.31903451681137085,
+      "rewards/margins": 9.262194633483887,
+      "rewards/rejected": -8.943160057067871,
+      "step": 800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.975127551020408e-07,
+      "logits/chosen": -2.0375688076019287,
+      "logits/rejected": -1.8554328680038452,
+      "logps/chosen": -321.64581298828125,
+      "logps/rejected": -150.6351776123047,
+      "loss": 0.0425,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.05941831320524216,
+      "rewards/margins": 8.272252082824707,
+      "rewards/rejected": -8.331669807434082,
+      "step": 810
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9432397959183673e-07,
+      "logits/chosen": -2.2465949058532715,
+      "logits/rejected": -2.003244638442993,
+      "logps/chosen": -339.7053527832031,
+      "logps/rejected": -173.89718627929688,
+      "loss": 0.0453,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.0983786582946777,
+      "rewards/margins": 8.536051750183105,
+      "rewards/rejected": -9.634429931640625,
+      "step": 820
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.911352040816326e-07,
+      "logits/chosen": -2.157315254211426,
+      "logits/rejected": -2.0112011432647705,
+      "logps/chosen": -285.68023681640625,
+      "logps/rejected": -163.58163452148438,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.5791069269180298,
+      "rewards/margins": 8.840615272521973,
+      "rewards/rejected": -9.419723510742188,
+      "step": 830
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.8794642857142854e-07,
+      "logits/chosen": -2.1603357791900635,
+      "logits/rejected": -1.983139991760254,
+      "logps/chosen": -275.4844665527344,
+      "logps/rejected": -165.48094177246094,
+      "loss": 0.036,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.849933922290802,
+      "rewards/margins": 8.592988967895508,
+      "rewards/rejected": -9.442923545837402,
+      "step": 840
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.8475765306122447e-07,
+      "logits/chosen": -2.1025400161743164,
+      "logits/rejected": -1.902698278427124,
+      "logps/chosen": -294.97662353515625,
+      "logps/rejected": -164.88980102539062,
+      "loss": 0.0255,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18203288316726685,
+      "rewards/margins": 9.839866638183594,
+      "rewards/rejected": -9.657832145690918,
+      "step": 850
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.815688775510204e-07,
+      "logits/chosen": -2.1174845695495605,
+      "logits/rejected": -1.9745981693267822,
+      "logps/chosen": -289.689208984375,
+      "logps/rejected": -225.04946899414062,
+      "loss": 0.0306,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.11574029922485352,
+      "rewards/margins": 14.800374031066895,
+      "rewards/rejected": -14.916116714477539,
+      "step": 860
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.783801020408163e-07,
+      "logits/chosen": -2.099398136138916,
+      "logits/rejected": -1.9381110668182373,
+      "logps/chosen": -280.7049255371094,
+      "logps/rejected": -162.89515686035156,
+      "loss": 0.0412,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6393367052078247,
+      "rewards/margins": 9.100909233093262,
+      "rewards/rejected": -9.740245819091797,
+      "step": 870
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.751913265306122e-07,
+      "logits/chosen": -2.130143404006958,
+      "logits/rejected": -1.8786506652832031,
+      "logps/chosen": -254.42440795898438,
+      "logps/rejected": -148.86289978027344,
+      "loss": 0.0416,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.5891353487968445,
+      "rewards/margins": 8.205097198486328,
+      "rewards/rejected": -8.794232368469238,
+      "step": 880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7200255102040814e-07,
+      "logits/chosen": -2.0689034461975098,
+      "logits/rejected": -2.004838466644287,
+      "logps/chosen": -253.21847534179688,
+      "logps/rejected": -173.26658630371094,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.9242547154426575,
+      "rewards/margins": 8.997157096862793,
+      "rewards/rejected": -9.921411514282227,
+      "step": 890
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6881377551020407e-07,
+      "logits/chosen": -2.24395751953125,
+      "logits/rejected": -2.0530552864074707,
+      "logps/chosen": -276.29327392578125,
+      "logps/rejected": -168.1978759765625,
+      "loss": 0.0338,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6147713661193848,
+      "rewards/margins": 9.07136058807373,
+      "rewards/rejected": -9.686132431030273,
+      "step": 900
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.65625e-07,
+      "logits/chosen": -2.1872873306274414,
+      "logits/rejected": -1.9537794589996338,
+      "logps/chosen": -313.37164306640625,
+      "logps/rejected": -169.32046508789062,
+      "loss": 0.0408,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.2682819664478302,
+      "rewards/margins": 9.430694580078125,
+      "rewards/rejected": -9.69897747039795,
+      "step": 910
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6243622448979593e-07,
+      "logits/chosen": -2.152787208557129,
+      "logits/rejected": -1.9356346130371094,
+      "logps/chosen": -280.98541259765625,
+      "logps/rejected": -152.12704467773438,
+      "loss": 0.0839,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.25039607286453247,
+      "rewards/margins": 8.020809173583984,
+      "rewards/rejected": -8.271204948425293,
+      "step": 920
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.592474489795918e-07,
+      "logits/chosen": -2.0773465633392334,
+      "logits/rejected": -1.9478585720062256,
+      "logps/chosen": -294.1889953613281,
+      "logps/rejected": -162.88656616210938,
+      "loss": 0.0256,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.10252566635608673,
+      "rewards/margins": 9.275411605834961,
+      "rewards/rejected": -9.377937316894531,
+      "step": 930
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5605867346938774e-07,
+      "logits/chosen": -2.1434168815612793,
+      "logits/rejected": -2.0151402950286865,
+      "logps/chosen": -255.9226531982422,
+      "logps/rejected": -164.32913208007812,
+      "loss": 0.033,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.7857500314712524,
+      "rewards/margins": 9.118104934692383,
+      "rewards/rejected": -9.903855323791504,
+      "step": 940
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5286989795918367e-07,
+      "logits/chosen": -2.177273750305176,
+      "logits/rejected": -2.035215139389038,
+      "logps/chosen": -275.280517578125,
+      "logps/rejected": -174.3971405029297,
+      "loss": 0.0274,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.5861722826957703,
+      "rewards/margins": 9.521924018859863,
+      "rewards/rejected": -10.1080961227417,
+      "step": 950
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4968112244897954e-07,
+      "logits/chosen": -2.123955011367798,
+      "logits/rejected": -1.9494431018829346,
+      "logps/chosen": -271.4372863769531,
+      "logps/rejected": -159.25759887695312,
+      "loss": 0.0183,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.16222012042999268,
+      "rewards/margins": 8.81199836730957,
+      "rewards/rejected": -8.974218368530273,
+      "step": 960
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.464923469387755e-07,
+      "logits/chosen": -2.1805062294006348,
+      "logits/rejected": -1.9929234981536865,
+      "logps/chosen": -312.1669006347656,
+      "logps/rejected": -173.0575714111328,
+      "loss": 0.0542,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.16012373566627502,
+      "rewards/margins": 10.075935363769531,
+      "rewards/rejected": -9.915812492370605,
+      "step": 970
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.433035714285714e-07,
+      "logits/chosen": -2.1818442344665527,
+      "logits/rejected": -1.9997507333755493,
+      "logps/chosen": -302.0064392089844,
+      "logps/rejected": -165.9043731689453,
+      "loss": 0.0448,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.39435887336730957,
+      "rewards/margins": 8.824365615844727,
+      "rewards/rejected": -9.21872329711914,
+      "step": 980
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.4011479591836734e-07,
+      "logits/chosen": -2.17393159866333,
+      "logits/rejected": -2.0282928943634033,
+      "logps/chosen": -297.4441833496094,
+      "logps/rejected": -168.88804626464844,
+      "loss": 0.0296,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8467293977737427,
+      "rewards/margins": 8.895944595336914,
+      "rewards/rejected": -9.742671966552734,
+      "step": 990
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3692602040816327e-07,
+      "logits/chosen": -2.210867166519165,
+      "logits/rejected": -1.9793678522109985,
+      "logps/chosen": -287.63763427734375,
+      "logps/rejected": -174.19747924804688,
+      "loss": 0.0362,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.5712664723396301,
+      "rewards/margins": 10.053550720214844,
+      "rewards/rejected": -10.62481689453125,
+      "step": 1000
+    },
+    {
+      "epoch": 0.57,
+      "eval_logits/chosen": -2.203035593032837,
+      "eval_logits/rejected": -2.0113065242767334,
+      "eval_logps/chosen": -303.8834228515625,
+      "eval_logps/rejected": -168.3771209716797,
+      "eval_loss": 0.04832219332456589,
+      "eval_rewards/accuracies": 0.9719827771186829,
+      "eval_rewards/chosen": -0.4980185925960541,
+      "eval_rewards/margins": 9.084371566772461,
+      "eval_rewards/rejected": -9.582389831542969,
+      "eval_runtime": 501.7343,
+      "eval_samples_per_second": 3.673,
+      "eval_steps_per_second": 0.116,
+      "step": 1000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3373724489795917e-07,
+      "logits/chosen": -2.2325656414031982,
+      "logits/rejected": -2.099182605743408,
+      "logps/chosen": -298.896240234375,
+      "logps/rejected": -178.80360412597656,
+      "loss": 0.0381,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.1191880702972412,
+      "rewards/margins": 8.851495742797852,
+      "rewards/rejected": -9.970685005187988,
+      "step": 1010
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.305484693877551e-07,
+      "logits/chosen": -2.1347708702087402,
+      "logits/rejected": -1.9578421115875244,
+      "logps/chosen": -310.70367431640625,
+      "logps/rejected": -173.8622283935547,
+      "loss": 0.022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.24546487629413605,
+      "rewards/margins": 9.734888076782227,
+      "rewards/rejected": -9.980352401733398,
+      "step": 1020
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2735969387755103e-07,
+      "logits/chosen": -2.114215135574341,
+      "logits/rejected": -1.9470380544662476,
+      "logps/chosen": -259.62371826171875,
+      "logps/rejected": -150.71444702148438,
+      "loss": 0.0263,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3662714660167694,
+      "rewards/margins": 8.222164154052734,
+      "rewards/rejected": -8.588435173034668,
+      "step": 1030
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.241709183673469e-07,
+      "logits/chosen": -2.0746610164642334,
+      "logits/rejected": -1.9087474346160889,
+      "logps/chosen": -308.8531799316406,
+      "logps/rejected": -173.00241088867188,
+      "loss": 0.0478,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.7547154426574707,
+      "rewards/margins": 9.060162544250488,
+      "rewards/rejected": -9.814878463745117,
+      "step": 1040
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2098214285714284e-07,
+      "logits/chosen": -2.1720070838928223,
+      "logits/rejected": -1.9714542627334595,
+      "logps/chosen": -282.46026611328125,
+      "logps/rejected": -165.5819549560547,
+      "loss": 0.0484,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8778654336929321,
+      "rewards/margins": 8.815220832824707,
+      "rewards/rejected": -9.693086624145508,
+      "step": 1050
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1779336734693877e-07,
+      "logits/chosen": -2.160717010498047,
+      "logits/rejected": -2.0338234901428223,
+      "logps/chosen": -293.3428649902344,
+      "logps/rejected": -180.75949096679688,
+      "loss": 0.0232,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8398637771606445,
+      "rewards/margins": 9.461033821105957,
+      "rewards/rejected": -10.300897598266602,
+      "step": 1060
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1460459183673467e-07,
+      "logits/chosen": -2.0859107971191406,
+      "logits/rejected": -1.9392955303192139,
+      "logps/chosen": -269.2712707519531,
+      "logps/rejected": -160.63858032226562,
+      "loss": 0.048,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.8251531720161438,
+      "rewards/margins": 8.415184020996094,
+      "rewards/rejected": -9.240337371826172,
+      "step": 1070
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.114158163265306e-07,
+      "logits/chosen": -2.114478349685669,
+      "logits/rejected": -2.0172407627105713,
+      "logps/chosen": -313.1733093261719,
+      "logps/rejected": -183.88912963867188,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37718772888183594,
+      "rewards/margins": 10.156351089477539,
+      "rewards/rejected": -10.533537864685059,
+      "step": 1080
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0822704081632654e-07,
+      "logits/chosen": -2.151115894317627,
+      "logits/rejected": -1.9639852046966553,
+      "logps/chosen": -304.9864196777344,
+      "logps/rejected": -179.6061248779297,
+      "loss": 0.0374,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8430510759353638,
+      "rewards/margins": 9.594018936157227,
+      "rewards/rejected": -10.4370698928833,
+      "step": 1090
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0503826530612244e-07,
+      "logits/chosen": -2.07926082611084,
+      "logits/rejected": -1.8545608520507812,
+      "logps/chosen": -287.17510986328125,
+      "logps/rejected": -175.82423400878906,
+      "loss": 0.025,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6869755387306213,
+      "rewards/margins": 9.874262809753418,
+      "rewards/rejected": -10.561239242553711,
+      "step": 1100
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 2.0184948979591837e-07,
+      "logits/chosen": -2.0271568298339844,
+      "logits/rejected": -1.9322795867919922,
+      "logps/chosen": -254.9891815185547,
+      "logps/rejected": -171.43470764160156,
+      "loss": 0.0285,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.40473413467407227,
+      "rewards/margins": 10.018217086791992,
+      "rewards/rejected": -10.422952651977539,
+      "step": 1110
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.9866071428571427e-07,
+      "logits/chosen": -1.9903011322021484,
+      "logits/rejected": -1.9317058324813843,
+      "logps/chosen": -239.4633026123047,
+      "logps/rejected": -167.33950805664062,
+      "loss": 0.052,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.950304627418518,
+      "rewards/margins": 8.423638343811035,
+      "rewards/rejected": -10.373943328857422,
+      "step": 1120
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9547193877551018e-07,
+      "logits/chosen": -2.1156675815582275,
+      "logits/rejected": -1.9115331172943115,
+      "logps/chosen": -283.9968566894531,
+      "logps/rejected": -171.2687225341797,
+      "loss": 0.0356,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6293619275093079,
+      "rewards/margins": 9.327872276306152,
+      "rewards/rejected": -9.957234382629395,
+      "step": 1130
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.922831632653061e-07,
+      "logits/chosen": -2.164762020111084,
+      "logits/rejected": -1.9720051288604736,
+      "logps/chosen": -332.24481201171875,
+      "logps/rejected": -172.13970947265625,
+      "loss": 0.0394,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.5399401783943176,
+      "rewards/margins": 8.87366008758545,
+      "rewards/rejected": -9.413599967956543,
+      "step": 1140
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.8909438775510204e-07,
+      "logits/chosen": -2.0545859336853027,
+      "logits/rejected": -1.916473627090454,
+      "logps/chosen": -279.626953125,
+      "logps/rejected": -176.50621032714844,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.018849492073059,
+      "rewards/margins": 9.826019287109375,
+      "rewards/rejected": -10.844868659973145,
+      "step": 1150
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.8590561224489794e-07,
+      "logits/chosen": -2.088841438293457,
+      "logits/rejected": -1.9257131814956665,
+      "logps/chosen": -292.3592834472656,
+      "logps/rejected": -173.84017944335938,
+      "loss": 0.0291,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.085963487625122,
+      "rewards/margins": 9.38125991821289,
+      "rewards/rejected": -10.467223167419434,
+      "step": 1160
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.8271683673469387e-07,
+      "logits/chosen": -2.1992461681365967,
+      "logits/rejected": -2.0129992961883545,
+      "logps/chosen": -298.34307861328125,
+      "logps/rejected": -182.71925354003906,
+      "loss": 0.039,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.1279481649398804,
+      "rewards/margins": 9.488454818725586,
+      "rewards/rejected": -10.61640453338623,
+      "step": 1170
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.795280612244898e-07,
+      "logits/chosen": -2.144819974899292,
+      "logits/rejected": -1.9659067392349243,
+      "logps/chosen": -256.8332214355469,
+      "logps/rejected": -152.0546112060547,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.4352855682373047,
+      "rewards/margins": 7.898017883300781,
+      "rewards/rejected": -8.33330249786377,
+      "step": 1180
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.763392857142857e-07,
+      "logits/chosen": -2.188812494277954,
+      "logits/rejected": -2.0105535984039307,
+      "logps/chosen": -311.55633544921875,
+      "logps/rejected": -167.11927795410156,
+      "loss": 0.0237,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.43777579069137573,
+      "rewards/margins": 8.915369987487793,
+      "rewards/rejected": -9.35314655303955,
+      "step": 1190
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.731505102040816e-07,
+      "logits/chosen": -2.087813377380371,
+      "logits/rejected": -1.9207570552825928,
+      "logps/chosen": -271.0974426269531,
+      "logps/rejected": -211.95523071289062,
+      "loss": 0.0268,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8373743295669556,
+      "rewards/margins": 13.245565414428711,
+      "rewards/rejected": -14.082939147949219,
+      "step": 1200
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.6996173469387754e-07,
+      "logits/chosen": -2.1115288734436035,
+      "logits/rejected": -1.9904811382293701,
+      "logps/chosen": -302.61138916015625,
+      "logps/rejected": -177.2365264892578,
+      "loss": 0.0235,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5768619775772095,
+      "rewards/margins": 8.965620040893555,
+      "rewards/rejected": -9.54248332977295,
+      "step": 1210
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.6677295918367345e-07,
+      "logits/chosen": -2.1301310062408447,
+      "logits/rejected": -1.879533052444458,
+      "logps/chosen": -332.2015686035156,
+      "logps/rejected": -165.51576232910156,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.5949792861938477,
+      "rewards/margins": 9.159207344055176,
+      "rewards/rejected": -9.754185676574707,
+      "step": 1220
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.6358418367346938e-07,
+      "logits/chosen": -2.1042232513427734,
+      "logits/rejected": -1.8513002395629883,
+      "logps/chosen": -344.7572937011719,
+      "logps/rejected": -172.65744018554688,
+      "loss": 0.0231,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.2580934464931488,
+      "rewards/margins": 9.699871063232422,
+      "rewards/rejected": -9.957964897155762,
+      "step": 1230
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.603954081632653e-07,
+      "logits/chosen": -2.068237066268921,
+      "logits/rejected": -1.8824779987335205,
+      "logps/chosen": -284.03509521484375,
+      "logps/rejected": -162.17001342773438,
+      "loss": 0.0275,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.403529554605484,
+      "rewards/margins": 9.167604446411133,
+      "rewards/rejected": -9.571134567260742,
+      "step": 1240
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.572066326530612e-07,
+      "logits/chosen": -2.1395645141601562,
+      "logits/rejected": -1.9513561725616455,
+      "logps/chosen": -342.67254638671875,
+      "logps/rejected": -174.70993041992188,
+      "loss": 0.0345,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07841818779706955,
+      "rewards/margins": 9.598522186279297,
+      "rewards/rejected": -9.676939964294434,
+      "step": 1250
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5401785714285714e-07,
+      "logits/chosen": -2.126796245574951,
+      "logits/rejected": -1.9961979389190674,
+      "logps/chosen": -316.7900695800781,
+      "logps/rejected": -176.53733825683594,
+      "loss": 0.0412,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.37025541067123413,
+      "rewards/margins": 9.903864860534668,
+      "rewards/rejected": -10.27411937713623,
+      "step": 1260
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.5082908163265307e-07,
+      "logits/chosen": -2.082777976989746,
+      "logits/rejected": -1.9228655099868774,
+      "logps/chosen": -297.30133056640625,
+      "logps/rejected": -166.17623901367188,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.2105169296264648,
+      "rewards/margins": 8.340936660766602,
+      "rewards/rejected": -9.551454544067383,
+      "step": 1270
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.4764030612244895e-07,
+      "logits/chosen": -2.1340670585632324,
+      "logits/rejected": -2.0120952129364014,
+      "logps/chosen": -306.83441162109375,
+      "logps/rejected": -174.4154510498047,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8796642422676086,
+      "rewards/margins": 9.489983558654785,
+      "rewards/rejected": -10.369647979736328,
+      "step": 1280
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4445153061224488e-07,
+      "logits/chosen": -2.180344581604004,
+      "logits/rejected": -2.054753541946411,
+      "logps/chosen": -319.8197326660156,
+      "logps/rejected": -179.91517639160156,
+      "loss": 0.0368,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.26657119393348694,
+      "rewards/margins": 10.043919563293457,
+      "rewards/rejected": -10.310490608215332,
+      "step": 1290
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.412627551020408e-07,
+      "logits/chosen": -2.139643907546997,
+      "logits/rejected": -1.9441308975219727,
+      "logps/chosen": -280.61492919921875,
+      "logps/rejected": -168.45120239257812,
+      "loss": 0.0245,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6699168086051941,
+      "rewards/margins": 9.325021743774414,
+      "rewards/rejected": -9.994939804077148,
+      "step": 1300
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3807397959183672e-07,
+      "logits/chosen": -2.2181859016418457,
+      "logits/rejected": -2.0422186851501465,
+      "logps/chosen": -348.22802734375,
+      "logps/rejected": -184.8292999267578,
+      "loss": 0.0389,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8054157495498657,
+      "rewards/margins": 9.742046356201172,
+      "rewards/rejected": -10.547462463378906,
+      "step": 1310
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.3488520408163265e-07,
+      "logits/chosen": -2.1537463665008545,
+      "logits/rejected": -1.9965778589248657,
+      "logps/chosen": -322.0302734375,
+      "logps/rejected": -177.57180786132812,
+      "loss": 0.039,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8039339780807495,
+      "rewards/margins": 9.590051651000977,
+      "rewards/rejected": -10.3939847946167,
+      "step": 1320
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.3169642857142858e-07,
+      "logits/chosen": -2.120084524154663,
+      "logits/rejected": -1.944972276687622,
+      "logps/chosen": -299.9647521972656,
+      "logps/rejected": -174.93545532226562,
+      "loss": 0.0365,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.0538573265075684,
+      "rewards/margins": 9.612497329711914,
+      "rewards/rejected": -10.666356086730957,
+      "step": 1330
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.2850765306122448e-07,
+      "logits/chosen": -2.0994505882263184,
+      "logits/rejected": -1.9836326837539673,
+      "logps/chosen": -327.00103759765625,
+      "logps/rejected": -176.6904754638672,
+      "loss": 0.0426,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0558558702468872,
+      "rewards/margins": 9.380393981933594,
+      "rewards/rejected": -10.436250686645508,
+      "step": 1340
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.253188775510204e-07,
+      "logits/chosen": -2.063891887664795,
+      "logits/rejected": -1.8763225078582764,
+      "logps/chosen": -245.1425018310547,
+      "logps/rejected": -155.8802032470703,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.3180797100067139,
+      "rewards/margins": 8.703597068786621,
+      "rewards/rejected": -10.021677017211914,
+      "step": 1350
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.2213010204081632e-07,
+      "logits/chosen": -2.1077721118927,
+      "logits/rejected": -1.9732635021209717,
+      "logps/chosen": -272.14886474609375,
+      "logps/rejected": -177.23715209960938,
+      "loss": 0.038,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5222896337509155,
+      "rewards/margins": 9.948299407958984,
+      "rewards/rejected": -10.470590591430664,
+      "step": 1360
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1894132653061225e-07,
+      "logits/chosen": -2.1442596912384033,
+      "logits/rejected": -1.995617151260376,
+      "logps/chosen": -297.60357666015625,
+      "logps/rejected": -176.26931762695312,
+      "loss": 0.0451,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.6867294311523438,
+      "rewards/margins": 8.880840301513672,
+      "rewards/rejected": -10.567569732666016,
+      "step": 1370
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1575255102040815e-07,
+      "logits/chosen": -2.1161892414093018,
+      "logits/rejected": -1.9788262844085693,
+      "logps/chosen": -291.28485107421875,
+      "logps/rejected": -171.2949676513672,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.612140417098999,
+      "rewards/margins": 8.652326583862305,
+      "rewards/rejected": -10.264467239379883,
+      "step": 1380
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1256377551020407e-07,
+      "logits/chosen": -2.1823105812072754,
+      "logits/rejected": -1.997688889503479,
+      "logps/chosen": -314.90081787109375,
+      "logps/rejected": -177.48538208007812,
+      "loss": 0.0348,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8158550262451172,
+      "rewards/margins": 9.316900253295898,
+      "rewards/rejected": -10.132753372192383,
+      "step": 1390
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.09375e-07,
+      "logits/chosen": -2.1604926586151123,
+      "logits/rejected": -1.92621648311615,
+      "logps/chosen": -322.7879638671875,
+      "logps/rejected": -174.97999572753906,
+      "loss": 0.0337,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6046790480613708,
+      "rewards/margins": 9.716318130493164,
+      "rewards/rejected": -10.32099723815918,
+      "step": 1400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0618622448979592e-07,
+      "logits/chosen": -2.1750383377075195,
+      "logits/rejected": -1.9606304168701172,
+      "logps/chosen": -304.9856872558594,
+      "logps/rejected": -166.8131103515625,
+      "loss": 0.0206,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.726199746131897,
+      "rewards/margins": 9.535404205322266,
+      "rewards/rejected": -10.261604309082031,
+      "step": 1410
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0299744897959183e-07,
+      "logits/chosen": -2.1826305389404297,
+      "logits/rejected": -1.9840447902679443,
+      "logps/chosen": -287.8218078613281,
+      "logps/rejected": -172.24215698242188,
+      "loss": 0.0326,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.0618003606796265,
+      "rewards/margins": 9.063058853149414,
+      "rewards/rejected": -10.124858856201172,
+      "step": 1420
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.980867346938775e-08,
+      "logits/chosen": -2.1716151237487793,
+      "logits/rejected": -2.0073437690734863,
+      "logps/chosen": -284.8751220703125,
+      "logps/rejected": -177.5775146484375,
+      "loss": 0.0339,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.5093313455581665,
+      "rewards/margins": 10.088600158691406,
+      "rewards/rejected": -10.597929954528809,
+      "step": 1430
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.661989795918367e-08,
+      "logits/chosen": -2.136314868927002,
+      "logits/rejected": -1.9978504180908203,
+      "logps/chosen": -299.4578857421875,
+      "logps/rejected": -177.2023468017578,
+      "loss": 0.0253,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.22061769664287567,
+      "rewards/margins": 10.269851684570312,
+      "rewards/rejected": -10.490468978881836,
+      "step": 1440
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.343112244897958e-08,
+      "logits/chosen": -2.1121840476989746,
+      "logits/rejected": -1.8750556707382202,
+      "logps/chosen": -294.89984130859375,
+      "logps/rejected": -169.1930389404297,
+      "loss": 0.0209,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6526461839675903,
+      "rewards/margins": 9.879607200622559,
+      "rewards/rejected": -10.532255172729492,
+      "step": 1450
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 9.024234693877552e-08,
+      "logits/chosen": -2.146242141723633,
+      "logits/rejected": -1.8776633739471436,
+      "logps/chosen": -313.9344177246094,
+      "logps/rejected": -166.5650177001953,
+      "loss": 0.0308,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.13873246312141418,
+      "rewards/margins": 9.890599250793457,
+      "rewards/rejected": -9.751866340637207,
+      "step": 1460
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.705357142857142e-08,
+      "logits/chosen": -2.079200506210327,
+      "logits/rejected": -1.8900413513183594,
+      "logps/chosen": -302.3304443359375,
+      "logps/rejected": -163.3623809814453,
+      "loss": 0.0126,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.21449720859527588,
+      "rewards/margins": 9.112278938293457,
+      "rewards/rejected": -9.326776504516602,
+      "step": 1470
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.386479591836734e-08,
+      "logits/chosen": -2.063764810562134,
+      "logits/rejected": -1.9468538761138916,
+      "logps/chosen": -271.62640380859375,
+      "logps/rejected": -183.78414916992188,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.5422567129135132,
+      "rewards/margins": 10.014627456665039,
+      "rewards/rejected": -10.556883811950684,
+      "step": 1480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.067602040816327e-08,
+      "logits/chosen": -2.0192930698394775,
+      "logits/rejected": -1.8542463779449463,
+      "logps/chosen": -282.27069091796875,
+      "logps/rejected": -164.059326171875,
+      "loss": 0.0239,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4797702729701996,
+      "rewards/margins": 9.287493705749512,
+      "rewards/rejected": -9.767263412475586,
+      "step": 1490
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.748724489795918e-08,
+      "logits/chosen": -2.097482204437256,
+      "logits/rejected": -1.9764598608016968,
+      "logps/chosen": -273.4100341796875,
+      "logps/rejected": -175.4436492919922,
+      "loss": 0.0318,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.9027210474014282,
+      "rewards/margins": 9.17738151550293,
+      "rewards/rejected": -10.080102920532227,
+      "step": 1500
+    },
+    {
+      "epoch": 0.86,
+      "eval_logits/chosen": -2.1461496353149414,
+      "eval_logits/rejected": -1.9506442546844482,
+      "eval_logps/chosen": -307.3617248535156,
+      "eval_logps/rejected": -178.54031372070312,
+      "eval_loss": 0.04423515126109123,
+      "eval_rewards/accuracies": 0.9719827771186829,
+      "eval_rewards/chosen": -0.8458480834960938,
+      "eval_rewards/margins": 9.752861022949219,
+      "eval_rewards/rejected": -10.598710060119629,
+      "eval_runtime": 498.2368,
+      "eval_samples_per_second": 3.699,
+      "eval_steps_per_second": 0.116,
+      "step": 1500
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.429846938775509e-08,
+      "logits/chosen": -2.1621012687683105,
+      "logits/rejected": -1.9208993911743164,
+      "logps/chosen": -343.2023010253906,
+      "logps/rejected": -189.607666015625,
+      "loss": 0.0304,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.39647844433784485,
+      "rewards/margins": 10.542058944702148,
+      "rewards/rejected": -10.938536643981934,
+      "step": 1510
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.110969387755102e-08,
+      "logits/chosen": -2.077564239501953,
+      "logits/rejected": -1.8879787921905518,
+      "logps/chosen": -285.5229797363281,
+      "logps/rejected": -171.03269958496094,
+      "loss": 0.0418,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.0820773839950562,
+      "rewards/margins": 9.324197769165039,
+      "rewards/rejected": -10.406275749206543,
+      "step": 1520
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.792091836734694e-08,
+      "logits/chosen": -2.1050283908843994,
+      "logits/rejected": -1.9014123678207397,
+      "logps/chosen": -291.99993896484375,
+      "logps/rejected": -167.6908721923828,
+      "loss": 0.0245,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.04504892975091934,
+      "rewards/margins": 10.022317886352539,
+      "rewards/rejected": -9.97726821899414,
+      "step": 1530
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.473214285714285e-08,
+      "logits/chosen": -2.155641794204712,
+      "logits/rejected": -1.9214122295379639,
+      "logps/chosen": -276.1907958984375,
+      "logps/rejected": -168.76535034179688,
+      "loss": 0.0403,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5811057090759277,
+      "rewards/margins": 9.711630821228027,
+      "rewards/rejected": -10.292736053466797,
+      "step": 1540
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.154336734693877e-08,
+      "logits/chosen": -2.0060572624206543,
+      "logits/rejected": -1.8284757137298584,
+      "logps/chosen": -251.60006713867188,
+      "logps/rejected": -161.85813903808594,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.079087495803833,
+      "rewards/margins": 8.86231803894043,
+      "rewards/rejected": -9.94140625,
+      "step": 1550
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.835459183673469e-08,
+      "logits/chosen": -2.1795308589935303,
+      "logits/rejected": -1.9668638706207275,
+      "logps/chosen": -307.10614013671875,
+      "logps/rejected": -198.31088256835938,
+      "loss": 0.0456,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.6056522130966187,
+      "rewards/margins": 12.019081115722656,
+      "rewards/rejected": -12.62473201751709,
+      "step": 1560
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.516581632653061e-08,
+      "logits/chosen": -1.972507119178772,
+      "logits/rejected": -1.8055419921875,
+      "logps/chosen": -260.49920654296875,
+      "logps/rejected": -156.6437225341797,
+      "loss": 0.0285,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8670014142990112,
+      "rewards/margins": 8.884390830993652,
+      "rewards/rejected": -9.751392364501953,
+      "step": 1570
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.197704081632653e-08,
+      "logits/chosen": -2.0382626056671143,
+      "logits/rejected": -1.8945573568344116,
+      "logps/chosen": -251.30612182617188,
+      "logps/rejected": -166.97303771972656,
+      "loss": 0.0475,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.7538485527038574,
+      "rewards/margins": 9.353151321411133,
+      "rewards/rejected": -10.107000350952148,
+      "step": 1580
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.878826530612245e-08,
+      "logits/chosen": -2.1863982677459717,
+      "logits/rejected": -1.9323434829711914,
+      "logps/chosen": -323.969970703125,
+      "logps/rejected": -168.3311767578125,
+      "loss": 0.0474,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.45526188611984253,
+      "rewards/margins": 9.516814231872559,
+      "rewards/rejected": -9.972076416015625,
+      "step": 1590
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.5599489795918364e-08,
+      "logits/chosen": -2.0845093727111816,
+      "logits/rejected": -1.8628953695297241,
+      "logps/chosen": -283.7400817871094,
+      "logps/rejected": -173.89602661132812,
+      "loss": 0.0331,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2650320529937744,
+      "rewards/margins": 10.87845516204834,
+      "rewards/rejected": -10.613424301147461,
+      "step": 1600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.241071428571429e-08,
+      "logits/chosen": -2.034493923187256,
+      "logits/rejected": -1.8942550420761108,
+      "logps/chosen": -270.16497802734375,
+      "logps/rejected": -166.28965759277344,
+      "loss": 0.0293,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.9170511960983276,
+      "rewards/margins": 9.314929962158203,
+      "rewards/rejected": -10.23198127746582,
+      "step": 1610
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.92219387755102e-08,
+      "logits/chosen": -2.102139949798584,
+      "logits/rejected": -1.9411756992340088,
+      "logps/chosen": -301.23846435546875,
+      "logps/rejected": -180.20574951171875,
+      "loss": 0.0467,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.1675564050674438,
+      "rewards/margins": 9.335586547851562,
+      "rewards/rejected": -10.503143310546875,
+      "step": 1620
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.603316326530612e-08,
+      "logits/chosen": -2.0670650005340576,
+      "logits/rejected": -1.8098779916763306,
+      "logps/chosen": -294.5066833496094,
+      "logps/rejected": -173.11416625976562,
+      "loss": 0.0246,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8192850351333618,
+      "rewards/margins": 9.408272743225098,
+      "rewards/rejected": -10.227557182312012,
+      "step": 1630
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.284438775510204e-08,
+      "logits/chosen": -2.138176918029785,
+      "logits/rejected": -1.978822112083435,
+      "logps/chosen": -329.6404724121094,
+      "logps/rejected": -190.4500274658203,
+      "loss": 0.0184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5538009405136108,
+      "rewards/margins": 10.36497688293457,
+      "rewards/rejected": -10.918777465820312,
+      "step": 1640
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.965561224489796e-08,
+      "logits/chosen": -2.1873650550842285,
+      "logits/rejected": -2.0006983280181885,
+      "logps/chosen": -301.3078308105469,
+      "logps/rejected": -175.1551055908203,
+      "loss": 0.0297,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.220243215560913,
+      "rewards/margins": 9.117000579833984,
+      "rewards/rejected": -10.337244033813477,
+      "step": 1650
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6466836734693878e-08,
+      "logits/chosen": -2.089918851852417,
+      "logits/rejected": -1.882301926612854,
+      "logps/chosen": -287.04632568359375,
+      "logps/rejected": -171.83804321289062,
+      "loss": 0.0308,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6570901274681091,
+      "rewards/margins": 9.935832023620605,
+      "rewards/rejected": -10.592921257019043,
+      "step": 1660
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3278061224489795e-08,
+      "logits/chosen": -2.073734998703003,
+      "logits/rejected": -1.8911638259887695,
+      "logps/chosen": -297.2498474121094,
+      "logps/rejected": -174.6929168701172,
+      "loss": 0.0237,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.06594938039779663,
+      "rewards/margins": 10.632802963256836,
+      "rewards/rejected": -10.566854476928711,
+      "step": 1670
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0089285714285716e-08,
+      "logits/chosen": -2.1300342082977295,
+      "logits/rejected": -2.0111982822418213,
+      "logps/chosen": -284.3525695800781,
+      "logps/rejected": -170.9279022216797,
+      "loss": 0.0392,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.1906980723142624,
+      "rewards/margins": 9.709856033325195,
+      "rewards/rejected": -9.900552749633789,
+      "step": 1680
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6900510204081633e-08,
+      "logits/chosen": -2.114060878753662,
+      "logits/rejected": -1.9764654636383057,
+      "logps/chosen": -276.199462890625,
+      "logps/rejected": -178.11782836914062,
+      "loss": 0.0337,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.643802285194397,
+      "rewards/margins": 9.700589179992676,
+      "rewards/rejected": -10.344388961791992,
+      "step": 1690
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.371173469387755e-08,
+      "logits/chosen": -1.897796392440796,
+      "logits/rejected": -1.800377607345581,
+      "logps/chosen": -251.58102416992188,
+      "logps/rejected": -162.783203125,
+      "loss": 0.0293,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.036834489554166794,
+      "rewards/margins": 9.945042610168457,
+      "rewards/rejected": -9.981878280639648,
+      "step": 1700
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0522959183673468e-08,
+      "logits/chosen": -2.0931904315948486,
+      "logits/rejected": -1.8743259906768799,
+      "logps/chosen": -295.89080810546875,
+      "logps/rejected": -169.799560546875,
+      "loss": 0.0309,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.7167753577232361,
+      "rewards/margins": 9.488900184631348,
+      "rewards/rejected": -10.205676078796387,
+      "step": 1710
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.334183673469387e-09,
+      "logits/chosen": -2.086829662322998,
+      "logits/rejected": -1.973637342453003,
+      "logps/chosen": -272.9532165527344,
+      "logps/rejected": -178.72866821289062,
+      "loss": 0.0283,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.3609329164028168,
+      "rewards/margins": 9.991233825683594,
+      "rewards/rejected": -10.352167129516602,
+      "step": 1720
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.145408163265306e-09,
+      "logits/chosen": -2.1046552658081055,
+      "logits/rejected": -1.9208564758300781,
+      "logps/chosen": -248.65567016601562,
+      "logps/rejected": -149.6916046142578,
+      "loss": 0.0201,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8961359262466431,
+      "rewards/margins": 8.393698692321777,
+      "rewards/rejected": -9.289834976196289,
+      "step": 1730
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.566326530612245e-10,
+      "logits/chosen": -2.1648898124694824,
+      "logits/rejected": -1.9197168350219727,
+      "logps/chosen": -293.1455078125,
+      "logps/rejected": -166.76507568359375,
+      "loss": 0.0269,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.503768265247345,
+      "rewards/margins": 9.358155250549316,
+      "rewards/rejected": -9.861923217773438,
+      "step": 1740
+    },
+    {
+      "epoch": 1.0,
+      "step": 1743,
+      "total_flos": 0.0,
+      "train_loss": 0.06308119606848633,
+      "train_runtime": 23531.0319,
+      "train_samples_per_second": 2.37,
+      "train_steps_per_second": 0.074
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1743,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5244e74363277f1c7a62dc098914bce5b3a39695ac66f66a71f6f7df61ed2f9
+size 5371