yiran-wang3
/

qwen1_chat_reflct_adamw_iter7

@@ -61,4 +61,4 @@ The following hyperparameters were used during training:
 - Transformers 4.45.0
 - Pytorch 2.4.0+cu121
 - Datasets 2.14.6
-- Tokenizers 0.20.2

 - Transformers 4.45.0
 - Pytorch 2.4.0+cu121
 - Datasets 2.14.6
+- Tokenizers 0.20.3

all_results.json CHANGED Viewed

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.44354424568322987,
-    "train_runtime": 174.5843,
     "train_samples": 3272,
-    "train_samples_per_second": 18.742,
-    "train_steps_per_second": 0.298
 }

 {
     "epoch": 1.0,
     "total_flos": 0.0,
+    "train_loss": 0.45137535952604735,
+    "train_runtime": 176.3389,
     "train_samples": 3272,
+    "train_samples_per_second": 18.555,
+    "train_steps_per_second": 0.295
 }

config.json CHANGED Viewed

@@ -23,7 +23,7 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.45.0",
-  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.45.0",
+  "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

train_results.json CHANGED Viewed

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.44354424568322987,
-    "train_runtime": 174.5843,
     "train_samples": 3272,
-    "train_samples_per_second": 18.742,
-    "train_steps_per_second": 0.298
 }

 {
     "epoch": 1.0,
     "total_flos": 0.0,
+    "train_loss": 0.45137535952604735,
+    "train_runtime": 176.3389,
     "train_samples": 3272,
+    "train_samples_per_second": 18.555,
+    "train_steps_per_second": 0.295
 }

trainer_state.json CHANGED Viewed

@@ -9,19 +9,19 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "debug/policy_chosen_logits": -1.2316170930862427,
-      "debug/policy_chosen_logps": -132.73162841796875,
-      "debug/policy_rejected_logits": -1.1728769540786743,
-      "debug/policy_rejected_logps": -162.23696899414062,
-      "debug/reference_chosen_logps": -132.73162841796875,
-      "debug/reference_rejected_logps": -162.23696899414062,
       "epoch": 0.019230769230769232,
-      "grad_norm": 21.661978971839787,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2316170930862427,
-      "logits/rejected": -1.1728769540786743,
-      "logps/chosen": -132.73162841796875,
-      "logps/rejected": -162.23696899414062,
       "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -30,1084 +30,1084 @@
       "step": 1
     },
     {
-      "debug/policy_chosen_logits": -1.2594889402389526,
-      "debug/policy_chosen_logps": -179.831298828125,
-      "debug/policy_rejected_logits": -1.1996418237686157,
-      "debug/policy_rejected_logps": -205.6752471923828,
-      "debug/reference_chosen_logps": -178.32009887695312,
-      "debug/reference_rejected_logps": -204.66696166992188,
       "epoch": 0.038461538461538464,
-      "grad_norm": 38.56925064024332,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2594889402389526,
-      "logits/rejected": -1.1996418237686157,
-      "logps/chosen": -179.831298828125,
-      "logps/rejected": -205.6752471923828,
-      "loss": 0.4876,
       "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.01511194184422493,
-      "rewards/margins": -0.005029010586440563,
-      "rewards/rejected": -0.010082930326461792,
       "step": 2
     },
     {
-      "debug/policy_chosen_logits": -1.2830009460449219,
-      "debug/policy_chosen_logps": -160.35537719726562,
-      "debug/policy_rejected_logits": -1.2497457265853882,
-      "debug/policy_rejected_logps": -240.45254516601562,
-      "debug/reference_chosen_logps": -153.9058837890625,
-      "debug/reference_rejected_logps": -233.42526245117188,
       "epoch": 0.057692307692307696,
-      "grad_norm": 74.62803863506201,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2830009460449219,
-      "logits/rejected": -1.2497457265853882,
-      "logps/chosen": -160.35537719726562,
-      "logps/rejected": -240.45254516601562,
-      "loss": 0.518,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.0644947960972786,
-      "rewards/margins": 0.005777954123914242,
-      "rewards/rejected": -0.07027274370193481,
       "step": 3
     },
     {
-      "debug/policy_chosen_logits": -1.1099666357040405,
-      "debug/policy_chosen_logps": -96.8184814453125,
-      "debug/policy_rejected_logits": -1.05377995967865,
-      "debug/policy_rejected_logps": -202.26805114746094,
-      "debug/reference_chosen_logps": -95.6224365234375,
-      "debug/reference_rejected_logps": -194.42694091796875,
       "epoch": 0.07692307692307693,
-      "grad_norm": 90.61433067087236,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1099666357040405,
-      "logits/rejected": -1.05377995967865,
-      "logps/chosen": -96.8184814453125,
-      "logps/rejected": -202.26805114746094,
-      "loss": 0.5302,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.011960487812757492,
-      "rewards/margins": 0.06645052880048752,
-      "rewards/rejected": -0.07841102033853531,
       "step": 4
     },
     {
-      "debug/policy_chosen_logits": -1.109797716140747,
-      "debug/policy_chosen_logps": -120.20796203613281,
-      "debug/policy_rejected_logits": -1.1077115535736084,
-      "debug/policy_rejected_logps": -216.43023681640625,
-      "debug/reference_chosen_logps": -118.03280639648438,
-      "debug/reference_rejected_logps": -212.2555694580078,
       "epoch": 0.09615384615384616,
-      "grad_norm": 59.296344658626325,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.109797716140747,
-      "logits/rejected": -1.1077115535736084,
-      "logps/chosen": -120.20796203613281,
-      "logps/rejected": -216.43023681640625,
-      "loss": 0.4996,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.02175154723227024,
-      "rewards/margins": 0.01999506726861,
-      "rewards/rejected": -0.04174661636352539,
       "step": 5
     },
     {
-      "debug/policy_chosen_logits": -1.3628556728363037,
-      "debug/policy_chosen_logps": -166.7021484375,
-      "debug/policy_rejected_logits": -1.273955225944519,
-      "debug/policy_rejected_logps": -198.93994140625,
-      "debug/reference_chosen_logps": -166.29620361328125,
-      "debug/reference_rejected_logps": -197.29959106445312,
       "epoch": 0.11538461538461539,
-      "grad_norm": 19.083922077983136,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.3628556728363037,
-      "logits/rejected": -1.273955225944519,
-      "logps/chosen": -166.7021484375,
-      "logps/rejected": -198.93994140625,
-      "loss": 0.4889,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.004059524275362492,
-      "rewards/margins": 0.012344006448984146,
-      "rewards/rejected": -0.016403531655669212,
       "step": 6
     },
     {
-      "debug/policy_chosen_logits": -1.2453309297561646,
-      "debug/policy_chosen_logps": -135.94847106933594,
-      "debug/policy_rejected_logits": -1.2954285144805908,
-      "debug/policy_rejected_logps": -176.56109619140625,
-      "debug/reference_chosen_logps": -136.97216796875,
-      "debug/reference_rejected_logps": -176.8516082763672,
       "epoch": 0.1346153846153846,
-      "grad_norm": 30.257774152774047,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2453309297561646,
-      "logits/rejected": -1.2954285144805908,
-      "logps/chosen": -135.94847106933594,
-      "logps/rejected": -176.56109619140625,
-      "loss": 0.4773,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.010236978530883789,
-      "rewards/margins": 0.0073319049552083015,
-      "rewards/rejected": 0.002905072644352913,
       "step": 7
     },
     {
-      "debug/policy_chosen_logits": -1.1769282817840576,
-      "debug/policy_chosen_logps": -130.25563049316406,
-      "debug/policy_rejected_logits": -1.251535177230835,
-      "debug/policy_rejected_logps": -211.3385009765625,
-      "debug/reference_chosen_logps": -135.37171936035156,
-      "debug/reference_rejected_logps": -218.43051147460938,
       "epoch": 0.15384615384615385,
-      "grad_norm": 53.6538734909917,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1769282817840576,
-      "logits/rejected": -1.251535177230835,
-      "logps/chosen": -130.25563049316406,
-      "logps/rejected": -211.3385009765625,
-      "loss": 0.4852,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.05116088315844536,
-      "rewards/margins": -0.019759180024266243,
-      "rewards/rejected": 0.07092006504535675,
       "step": 8
     },
     {
-      "debug/policy_chosen_logits": -1.1520813703536987,
-      "debug/policy_chosen_logps": -130.01480102539062,
-      "debug/policy_rejected_logits": -1.0665879249572754,
-      "debug/policy_rejected_logps": -164.55233764648438,
-      "debug/reference_chosen_logps": -131.8277130126953,
-      "debug/reference_rejected_logps": -168.2589569091797,
       "epoch": 0.17307692307692307,
-      "grad_norm": 68.56866910167538,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1520813703536987,
-      "logits/rejected": -1.0665879249572754,
-      "logps/chosen": -130.01480102539062,
-      "logps/rejected": -164.55233764648438,
-      "loss": 0.4662,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.018129272386431694,
-      "rewards/margins": -0.018936950713396072,
-      "rewards/rejected": 0.03706622123718262,
       "step": 9
     },
     {
-      "debug/policy_chosen_logits": -1.2677048444747925,
-      "debug/policy_chosen_logps": -108.82756042480469,
-      "debug/policy_rejected_logits": -1.1619057655334473,
-      "debug/policy_rejected_logps": -212.99595642089844,
-      "debug/reference_chosen_logps": -115.01768493652344,
-      "debug/reference_rejected_logps": -210.9394073486328,
       "epoch": 0.19230769230769232,
-      "grad_norm": 29.961533234795134,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2677048444747925,
-      "logits/rejected": -1.1619057655334473,
-      "logps/chosen": -108.82756042480469,
-      "logps/rejected": -212.99595642089844,
-      "loss": 0.4517,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.061901286244392395,
-      "rewards/margins": 0.0824667438864708,
-      "rewards/rejected": -0.020565450191497803,
       "step": 10
     },
     {
-      "debug/policy_chosen_logits": -1.229325532913208,
-      "debug/policy_chosen_logps": -152.08340454101562,
-      "debug/policy_rejected_logits": -1.1806657314300537,
-      "debug/policy_rejected_logps": -284.9648132324219,
-      "debug/reference_chosen_logps": -145.8704376220703,
-      "debug/reference_rejected_logps": -278.4148254394531,
       "epoch": 0.21153846153846154,
-      "grad_norm": 42.197936495884626,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.229325532913208,
-      "logits/rejected": -1.1806657314300537,
-      "logps/chosen": -152.08340454101562,
-      "logps/rejected": -284.9648132324219,
-      "loss": 0.4869,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.0621296763420105,
-      "rewards/margins": 0.0033701807260513306,
-      "rewards/rejected": -0.06549985706806183,
       "step": 11
     },
     {
-      "debug/policy_chosen_logits": -0.962820291519165,
-      "debug/policy_chosen_logps": -134.88470458984375,
-      "debug/policy_rejected_logits": -1.1076884269714355,
-      "debug/policy_rejected_logps": -276.01092529296875,
-      "debug/reference_chosen_logps": -137.8263397216797,
-      "debug/reference_rejected_logps": -271.51300048828125,
       "epoch": 0.23076923076923078,
-      "grad_norm": 19.480574052456717,
       "learning_rate": 1e-06,
-      "logits/chosen": -0.962820291519165,
-      "logits/rejected": -1.1076884269714355,
-      "logps/chosen": -134.88470458984375,
-      "logps/rejected": -276.01092529296875,
-      "loss": 0.4631,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.029416313394904137,
-      "rewards/margins": 0.07439534366130829,
-      "rewards/rejected": -0.0449790358543396,
       "step": 12
     },
     {
-      "debug/policy_chosen_logits": -1.187856912612915,
-      "debug/policy_chosen_logps": -136.24522399902344,
-      "debug/policy_rejected_logits": -1.2246490716934204,
-      "debug/policy_rejected_logps": -233.98336791992188,
-      "debug/reference_chosen_logps": -139.45712280273438,
-      "debug/reference_rejected_logps": -229.1366729736328,
       "epoch": 0.25,
-      "grad_norm": 56.94222632531445,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.187856912612915,
-      "logits/rejected": -1.2246490716934204,
-      "logps/chosen": -136.24522399902344,
-      "logps/rejected": -233.98336791992188,
-      "loss": 0.4664,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.03211888298392296,
-      "rewards/margins": 0.08058580756187439,
-      "rewards/rejected": -0.04846692830324173,
       "step": 13
     },
     {
-      "debug/policy_chosen_logits": -1.2769882678985596,
-      "debug/policy_chosen_logps": -104.90899658203125,
-      "debug/policy_rejected_logits": -1.2562098503112793,
-      "debug/policy_rejected_logps": -237.41644287109375,
-      "debug/reference_chosen_logps": -109.57907104492188,
-      "debug/reference_rejected_logps": -236.04598999023438,
       "epoch": 0.2692307692307692,
-      "grad_norm": 24.76649234851808,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2769882678985596,
-      "logits/rejected": -1.2562098503112793,
-      "logps/chosen": -104.90899658203125,
-      "logps/rejected": -237.41644287109375,
-      "loss": 0.4384,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.04670079052448273,
-      "rewards/margins": 0.06040526181459427,
-      "rewards/rejected": -0.013704471290111542,
       "step": 14
     },
     {
-      "debug/policy_chosen_logits": -1.2544196844100952,
-      "debug/policy_chosen_logps": -109.65789031982422,
-      "debug/policy_rejected_logits": -1.310359001159668,
-      "debug/policy_rejected_logps": -199.24005126953125,
-      "debug/reference_chosen_logps": -108.41796875,
-      "debug/reference_rejected_logps": -197.53604125976562,
       "epoch": 0.28846153846153844,
-      "grad_norm": 59.48873508478175,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2544196844100952,
-      "logits/rejected": -1.310359001159668,
-      "logps/chosen": -109.65789031982422,
-      "logps/rejected": -199.24005126953125,
-      "loss": 0.4714,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.012399187311530113,
-      "rewards/margins": 0.004640864208340645,
-      "rewards/rejected": -0.017040051519870758,
       "step": 15
     },
     {
-      "debug/policy_chosen_logits": -1.2345539331436157,
-      "debug/policy_chosen_logps": -149.3842010498047,
-      "debug/policy_rejected_logits": -1.1627442836761475,
-      "debug/policy_rejected_logps": -241.727294921875,
-      "debug/reference_chosen_logps": -145.96755981445312,
-      "debug/reference_rejected_logps": -233.75701904296875,
       "epoch": 0.3076923076923077,
-      "grad_norm": 54.175150299747976,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2345539331436157,
-      "logits/rejected": -1.1627442836761475,
-      "logps/chosen": -149.3842010498047,
-      "logps/rejected": -241.727294921875,
-      "loss": 0.4362,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.034166477620601654,
-      "rewards/margins": 0.0455363467335701,
-      "rewards/rejected": -0.07970283180475235,
       "step": 16
     },
     {
-      "debug/policy_chosen_logits": -1.2490094900131226,
-      "debug/policy_chosen_logps": -146.42208862304688,
-      "debug/policy_rejected_logits": -1.1850720643997192,
-      "debug/policy_rejected_logps": -166.25057983398438,
-      "debug/reference_chosen_logps": -150.5763397216797,
-      "debug/reference_rejected_logps": -161.2530975341797,
       "epoch": 0.3269230769230769,
-      "grad_norm": 33.4444193325989,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2490094900131226,
-      "logits/rejected": -1.1850720643997192,
-      "logps/chosen": -146.42208862304688,
-      "logps/rejected": -166.25057983398438,
-      "loss": 0.4438,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.04154255986213684,
-      "rewards/margins": 0.09151734411716461,
-      "rewards/rejected": -0.04997478425502777,
       "step": 17
     },
     {
-      "debug/policy_chosen_logits": -1.2004398107528687,
-      "debug/policy_chosen_logps": -163.50543212890625,
-      "debug/policy_rejected_logits": -1.204341173171997,
-      "debug/policy_rejected_logps": -288.0180358886719,
-      "debug/reference_chosen_logps": -163.77049255371094,
-      "debug/reference_rejected_logps": -263.1712646484375,
       "epoch": 0.34615384615384615,
-      "grad_norm": 61.98069348299531,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2004398107528687,
-      "logits/rejected": -1.204341173171997,
-      "logps/chosen": -163.50543212890625,
-      "logps/rejected": -288.0180358886719,
-      "loss": 0.4592,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.002650529146194458,
-      "rewards/margins": 0.251118004322052,
-      "rewards/rejected": -0.24846749007701874,
       "step": 18
     },
     {
-      "debug/policy_chosen_logits": -1.3331176042556763,
-      "debug/policy_chosen_logps": -110.75132751464844,
-      "debug/policy_rejected_logits": -1.2217522859573364,
-      "debug/policy_rejected_logps": -207.45452880859375,
-      "debug/reference_chosen_logps": -115.902587890625,
-      "debug/reference_rejected_logps": -205.65411376953125,
       "epoch": 0.36538461538461536,
-      "grad_norm": 20.11730700354406,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.3331176042556763,
-      "logits/rejected": -1.2217522859573364,
-      "logps/chosen": -110.75132751464844,
-      "logps/rejected": -207.45452880859375,
-      "loss": 0.4491,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.05151257663965225,
-      "rewards/margins": 0.06951689720153809,
-      "rewards/rejected": -0.018004322424530983,
       "step": 19
     },
     {
-      "debug/policy_chosen_logits": -1.338209629058838,
-      "debug/policy_chosen_logps": -144.75852966308594,
-      "debug/policy_rejected_logits": -1.3235102891921997,
-      "debug/policy_rejected_logps": -262.59710693359375,
-      "debug/reference_chosen_logps": -141.7485809326172,
-      "debug/reference_rejected_logps": -245.33224487304688,
       "epoch": 0.38461538461538464,
-      "grad_norm": 40.754402513433696,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.338209629058838,
-      "logits/rejected": -1.3235102891921997,
-      "logps/chosen": -144.75852966308594,
-      "logps/rejected": -262.59710693359375,
-      "loss": 0.4624,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.03009958378970623,
-      "rewards/margins": 0.14254869520664215,
-      "rewards/rejected": -0.17264828085899353,
       "step": 20
     },
     {
-      "debug/policy_chosen_logits": -1.2065143585205078,
-      "debug/policy_chosen_logps": -146.51068115234375,
-      "debug/policy_rejected_logits": -1.2265596389770508,
-      "debug/policy_rejected_logps": -175.40896606445312,
-      "debug/reference_chosen_logps": -140.78436279296875,
-      "debug/reference_rejected_logps": -168.80673217773438,
       "epoch": 0.40384615384615385,
-      "grad_norm": 22.110741992294734,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2065143585205078,
-      "logits/rejected": -1.2265596389770508,
-      "logps/chosen": -146.51068115234375,
-      "logps/rejected": -175.40896606445312,
-      "loss": 0.4701,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.05726320296525955,
-      "rewards/margins": 0.008759044110774994,
-      "rewards/rejected": -0.06602225452661514,
       "step": 21
     },
     {
-      "debug/policy_chosen_logits": -1.2565746307373047,
-      "debug/policy_chosen_logps": -158.69158935546875,
-      "debug/policy_rejected_logits": -1.1477868556976318,
-      "debug/policy_rejected_logps": -176.73370361328125,
-      "debug/reference_chosen_logps": -156.03787231445312,
-      "debug/reference_rejected_logps": -172.10971069335938,
       "epoch": 0.4230769230769231,
-      "grad_norm": 44.62287437323654,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2565746307373047,
-      "logits/rejected": -1.1477868556976318,
-      "logps/chosen": -158.69158935546875,
-      "logps/rejected": -176.73370361328125,
-      "loss": 0.4407,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.026537198573350906,
-      "rewards/margins": 0.019702596589922905,
-      "rewards/rejected": -0.04623979702591896,
       "step": 22
     },
     {
-      "debug/policy_chosen_logits": -1.1006157398223877,
-      "debug/policy_chosen_logps": -100.4822998046875,
-      "debug/policy_rejected_logits": -1.035710096359253,
-      "debug/policy_rejected_logps": -333.0516357421875,
-      "debug/reference_chosen_logps": -104.65483093261719,
-      "debug/reference_rejected_logps": -310.935302734375,
       "epoch": 0.4423076923076923,
-      "grad_norm": 24.118006233865607,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1006157398223877,
-      "logits/rejected": -1.035710096359253,
-      "logps/chosen": -100.4822998046875,
-      "logps/rejected": -333.0516357421875,
-      "loss": 0.4535,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.04172533005475998,
-      "rewards/margins": 0.26288849115371704,
-      "rewards/rejected": -0.22116313874721527,
       "step": 23
     },
     {
-      "debug/policy_chosen_logits": -1.350253701210022,
-      "debug/policy_chosen_logps": -149.94215393066406,
-      "debug/policy_rejected_logits": -1.2977714538574219,
-      "debug/policy_rejected_logps": -326.00872802734375,
-      "debug/reference_chosen_logps": -150.9017333984375,
-      "debug/reference_rejected_logps": -303.9064636230469,
       "epoch": 0.46153846153846156,
-      "grad_norm": 31.59608650044171,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.350253701210022,
-      "logits/rejected": -1.2977714538574219,
-      "logps/chosen": -149.94215393066406,
-      "logps/rejected": -326.00872802734375,
-      "loss": 0.4305,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.00959576666355133,
-      "rewards/margins": 0.23061853647232056,
-      "rewards/rejected": -0.22102276980876923,
       "step": 24
     },
     {
-      "debug/policy_chosen_logits": -1.2466541528701782,
-      "debug/policy_chosen_logps": -179.1575164794922,
-      "debug/policy_rejected_logits": -1.2147475481033325,
-      "debug/policy_rejected_logps": -295.2662658691406,
-      "debug/reference_chosen_logps": -180.40638732910156,
-      "debug/reference_rejected_logps": -281.09112548828125,
       "epoch": 0.4807692307692308,
-      "grad_norm": 47.73909960510058,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2466541528701782,
-      "logits/rejected": -1.2147475481033325,
-      "logps/chosen": -179.1575164794922,
-      "logps/rejected": -295.2662658691406,
-      "loss": 0.4171,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.012488747015595436,
-      "rewards/margins": 0.1542397141456604,
-      "rewards/rejected": -0.14175096154212952,
       "step": 25
     },
     {
-      "debug/policy_chosen_logits": -1.204865574836731,
-      "debug/policy_chosen_logps": -137.5517578125,
-      "debug/policy_rejected_logits": -1.3782544136047363,
-      "debug/policy_rejected_logps": -208.717041015625,
-      "debug/reference_chosen_logps": -143.9291229248047,
-      "debug/reference_rejected_logps": -212.34486389160156,
       "epoch": 0.5,
-      "grad_norm": 41.22809898372289,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.204865574836731,
-      "logits/rejected": -1.3782544136047363,
-      "logps/chosen": -137.5517578125,
-      "logps/rejected": -208.717041015625,
-      "loss": 0.4209,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.06377358734607697,
-      "rewards/margins": 0.027495335787534714,
-      "rewards/rejected": 0.03627825155854225,
       "step": 26
     },
     {
-      "debug/policy_chosen_logits": -1.2046425342559814,
-      "debug/policy_chosen_logps": -140.90965270996094,
-      "debug/policy_rejected_logits": -1.2170807123184204,
-      "debug/policy_rejected_logps": -170.78704833984375,
-      "debug/reference_chosen_logps": -140.62596130371094,
-      "debug/reference_rejected_logps": -174.9475860595703,
       "epoch": 0.5192307692307693,
-      "grad_norm": 15.283751437103401,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2046425342559814,
-      "logits/rejected": -1.2170807123184204,
-      "logps/chosen": -140.90965270996094,
-      "logps/rejected": -170.78704833984375,
-      "loss": 0.4228,
-      "rewards/accuracies": 0.125,
-      "rewards/chosen": -0.002836771309375763,
-      "rewards/margins": -0.04444221407175064,
-      "rewards/rejected": 0.04160544276237488,
       "step": 27
     },
     {
-      "debug/policy_chosen_logits": -1.4107595682144165,
-      "debug/policy_chosen_logps": -130.7558135986328,
-      "debug/policy_rejected_logits": -1.4000462293624878,
-      "debug/policy_rejected_logps": -164.4125213623047,
-      "debug/reference_chosen_logps": -137.77110290527344,
-      "debug/reference_rejected_logps": -167.27540588378906,
       "epoch": 0.5384615384615384,
-      "grad_norm": 38.338385484834546,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.4107595682144165,
-      "logits/rejected": -1.4000462293624878,
-      "logps/chosen": -130.7558135986328,
-      "logps/rejected": -164.4125213623047,
-      "loss": 0.4467,
-      "rewards/accuracies": 0.25,
-      "rewards/chosen": 0.07015287131071091,
-      "rewards/margins": 0.04152403771877289,
-      "rewards/rejected": 0.02862883359193802,
       "step": 28
     },
     {
-      "debug/policy_chosen_logits": -1.398485779762268,
-      "debug/policy_chosen_logps": -150.90823364257812,
-      "debug/policy_rejected_logits": -1.174936294555664,
-      "debug/policy_rejected_logps": -291.80194091796875,
-      "debug/reference_chosen_logps": -158.24673461914062,
-      "debug/reference_rejected_logps": -276.33837890625,
       "epoch": 0.5576923076923077,
-      "grad_norm": 15.468395775938875,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.398485779762268,
-      "logits/rejected": -1.174936294555664,
-      "logps/chosen": -150.90823364257812,
-      "logps/rejected": -291.80194091796875,
-      "loss": 0.4396,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.07338497042655945,
-      "rewards/margins": 0.22802035510540009,
-      "rewards/rejected": -0.15463536977767944,
       "step": 29
     },
     {
-      "debug/policy_chosen_logits": -1.336907982826233,
-      "debug/policy_chosen_logps": -133.89599609375,
-      "debug/policy_rejected_logits": -1.2603211402893066,
-      "debug/policy_rejected_logps": -196.19960021972656,
-      "debug/reference_chosen_logps": -142.37545776367188,
-      "debug/reference_rejected_logps": -191.63192749023438,
       "epoch": 0.5769230769230769,
-      "grad_norm": 29.826267277718433,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.336907982826233,
-      "logits/rejected": -1.2603211402893066,
-      "logps/chosen": -133.89599609375,
-      "logps/rejected": -196.19960021972656,
-      "loss": 0.4405,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.08479461818933487,
-      "rewards/margins": 0.1304713636636734,
-      "rewards/rejected": -0.04567674547433853,
       "step": 30
     },
     {
-      "debug/policy_chosen_logits": -1.3420963287353516,
-      "debug/policy_chosen_logps": -101.8618392944336,
-      "debug/policy_rejected_logits": -1.1649795770645142,
-      "debug/policy_rejected_logps": -241.93505859375,
-      "debug/reference_chosen_logps": -110.1383056640625,
-      "debug/reference_rejected_logps": -240.0742950439453,
       "epoch": 0.5961538461538461,
-      "grad_norm": 27.431269846469586,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.3420963287353516,
-      "logits/rejected": -1.1649795770645142,
-      "logps/chosen": -101.8618392944336,
-      "logps/rejected": -241.93505859375,
-      "loss": 0.468,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.08276471495628357,
-      "rewards/margins": 0.10137245059013367,
-      "rewards/rejected": -0.0186077281832695,
       "step": 31
     },
     {
-      "debug/policy_chosen_logits": -1.2078020572662354,
-      "debug/policy_chosen_logps": -119.91991424560547,
-      "debug/policy_rejected_logits": -0.9945322871208191,
-      "debug/policy_rejected_logps": -330.7967529296875,
-      "debug/reference_chosen_logps": -139.70213317871094,
-      "debug/reference_rejected_logps": -319.4359130859375,
       "epoch": 0.6153846153846154,
-      "grad_norm": 14.75813816004449,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2078020572662354,
-      "logits/rejected": -0.9945322871208191,
-      "logps/chosen": -119.91991424560547,
-      "logps/rejected": -330.7967529296875,
-      "loss": 0.4,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.1978221833705902,
-      "rewards/margins": 0.3114301562309265,
-      "rewards/rejected": -0.11360795795917511,
       "step": 32
     },
     {
-      "debug/policy_chosen_logits": -1.1920416355133057,
-      "debug/policy_chosen_logps": -98.884765625,
-      "debug/policy_rejected_logits": -1.2546271085739136,
-      "debug/policy_rejected_logps": -283.1683654785156,
-      "debug/reference_chosen_logps": -110.47108459472656,
-      "debug/reference_rejected_logps": -274.7044677734375,
       "epoch": 0.6346153846153846,
-      "grad_norm": 17.792135410719844,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1920416355133057,
-      "logits/rejected": -1.2546271085739136,
-      "logps/chosen": -98.884765625,
-      "logps/rejected": -283.1683654785156,
-      "loss": 0.404,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.11586315184831619,
-      "rewards/margins": 0.20050224661827087,
-      "rewards/rejected": -0.08463907986879349,
       "step": 33
     },
     {
-      "debug/policy_chosen_logits": -1.32485830783844,
-      "debug/policy_chosen_logps": -133.64706420898438,
-      "debug/policy_rejected_logits": -1.2567251920700073,
-      "debug/policy_rejected_logps": -243.16220092773438,
-      "debug/reference_chosen_logps": -137.5300750732422,
-      "debug/reference_rejected_logps": -242.2387237548828,
       "epoch": 0.6538461538461539,
-      "grad_norm": 76.76926374768559,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.32485830783844,
-      "logits/rejected": -1.2567251920700073,
-      "logps/chosen": -133.64706420898438,
-      "logps/rejected": -243.16220092773438,
-      "loss": 0.4264,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.03883013129234314,
-      "rewards/margins": 0.04806497320532799,
-      "rewards/rejected": -0.009234847500920296,
       "step": 34
     },
     {
-      "debug/policy_chosen_logits": -1.3281021118164062,
-      "debug/policy_chosen_logps": -114.21416473388672,
-      "debug/policy_rejected_logits": -1.1842998266220093,
-      "debug/policy_rejected_logps": -230.96640014648438,
-      "debug/reference_chosen_logps": -126.98857116699219,
-      "debug/reference_rejected_logps": -227.5498809814453,
       "epoch": 0.6730769230769231,
-      "grad_norm": 16.48513566801007,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.3281021118164062,
-      "logits/rejected": -1.1842998266220093,
-      "logps/chosen": -114.21416473388672,
-      "logps/rejected": -230.96640014648438,
-      "loss": 0.4124,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.12774410843849182,
-      "rewards/margins": 0.1619093120098114,
-      "rewards/rejected": -0.03416522219777107,
       "step": 35
     },
     {
-      "debug/policy_chosen_logits": -1.1566346883773804,
-      "debug/policy_chosen_logps": -117.29952239990234,
-      "debug/policy_rejected_logits": -1.0660278797149658,
-      "debug/policy_rejected_logps": -323.2428894042969,
-      "debug/reference_chosen_logps": -125.93283081054688,
-      "debug/reference_rejected_logps": -305.2673645019531,
       "epoch": 0.6923076923076923,
-      "grad_norm": 17.26263202281334,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1566346883773804,
-      "logits/rejected": -1.0660278797149658,
-      "logps/chosen": -117.29952239990234,
-      "logps/rejected": -323.2428894042969,
-      "loss": 0.4024,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.08633305877447128,
-      "rewards/margins": 0.2660883069038391,
-      "rewards/rejected": -0.17975522577762604,
       "step": 36
     },
     {
-      "debug/policy_chosen_logits": -1.2661734819412231,
-      "debug/policy_chosen_logps": -139.73275756835938,
-      "debug/policy_rejected_logits": -1.219040036201477,
-      "debug/policy_rejected_logps": -192.42286682128906,
-      "debug/reference_chosen_logps": -141.48370361328125,
-      "debug/reference_rejected_logps": -179.03509521484375,
       "epoch": 0.7115384615384616,
-      "grad_norm": 31.112423756254753,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2661734819412231,
-      "logits/rejected": -1.219040036201477,
-      "logps/chosen": -139.73275756835938,
-      "logps/rejected": -192.42286682128906,
-      "loss": 0.4352,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.017509642988443375,
-      "rewards/margins": 0.15138748288154602,
-      "rewards/rejected": -0.13387782871723175,
       "step": 37
     },
     {
-      "debug/policy_chosen_logits": -1.4617388248443604,
-      "debug/policy_chosen_logps": -119.26033020019531,
-      "debug/policy_rejected_logits": -1.4288504123687744,
-      "debug/policy_rejected_logps": -181.7017364501953,
-      "debug/reference_chosen_logps": -127.11656188964844,
-      "debug/reference_rejected_logps": -176.86419677734375,
       "epoch": 0.7307692307692307,
-      "grad_norm": 42.01899889081456,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.4617388248443604,
-      "logits/rejected": -1.4288504123687744,
-      "logps/chosen": -119.26033020019531,
-      "logps/rejected": -181.7017364501953,
-      "loss": 0.3721,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.07856231927871704,
-      "rewards/margins": 0.12693758308887482,
-      "rewards/rejected": -0.04837527871131897,
       "step": 38
     },
     {
-      "debug/policy_chosen_logits": -1.2514652013778687,
-      "debug/policy_chosen_logps": -214.69508361816406,
-      "debug/policy_rejected_logits": -1.1772401332855225,
-      "debug/policy_rejected_logps": -197.77090454101562,
-      "debug/reference_chosen_logps": -205.87307739257812,
-      "debug/reference_rejected_logps": -191.47265625,
       "epoch": 0.75,
-      "grad_norm": 22.111384375060176,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2514652013778687,
-      "logits/rejected": -1.1772401332855225,
-      "logps/chosen": -214.69508361816406,
-      "logps/rejected": -197.77090454101562,
-      "loss": 0.4443,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.08822001516819,
-      "rewards/margins": -0.02523757889866829,
-      "rewards/rejected": -0.06298243254423141,
       "step": 39
     },
     {
-      "debug/policy_chosen_logits": -1.4284347295761108,
-      "debug/policy_chosen_logps": -147.825927734375,
-      "debug/policy_rejected_logits": -1.3076362609863281,
-      "debug/policy_rejected_logps": -254.9888153076172,
-      "debug/reference_chosen_logps": -146.136962890625,
-      "debug/reference_rejected_logps": -221.61865234375,
       "epoch": 0.7692307692307693,
-      "grad_norm": 78.31350758335499,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.4284347295761108,
-      "logits/rejected": -1.3076362609863281,
-      "logps/chosen": -147.825927734375,
-      "logps/rejected": -254.9888153076172,
-      "loss": 0.4668,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.016889560967683792,
-      "rewards/margins": 0.316812127828598,
-      "rewards/rejected": -0.3337016999721527,
       "step": 40
     },
     {
-      "debug/policy_chosen_logits": -1.2104414701461792,
-      "debug/policy_chosen_logps": -180.59503173828125,
-      "debug/policy_rejected_logits": -1.1626763343811035,
-      "debug/policy_rejected_logps": -299.04248046875,
-      "debug/reference_chosen_logps": -183.44879150390625,
-      "debug/reference_rejected_logps": -268.26422119140625,
       "epoch": 0.7884615384615384,
-      "grad_norm": 50.194451097738806,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2104414701461792,
-      "logits/rejected": -1.1626763343811035,
-      "logps/chosen": -180.59503173828125,
-      "logps/rejected": -299.04248046875,
-      "loss": 0.3985,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.02853771299123764,
-      "rewards/margins": 0.3363204598426819,
-      "rewards/rejected": -0.30778273940086365,
       "step": 41
     },
     {
-      "debug/policy_chosen_logits": -1.2823656797409058,
-      "debug/policy_chosen_logps": -105.43904876708984,
-      "debug/policy_rejected_logits": -1.2949018478393555,
-      "debug/policy_rejected_logps": -217.46287536621094,
-      "debug/reference_chosen_logps": -116.0798568725586,
-      "debug/reference_rejected_logps": -202.04270935058594,
       "epoch": 0.8076923076923077,
-      "grad_norm": 71.44356233819985,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2823656797409058,
-      "logits/rejected": -1.2949018478393555,
-      "logps/chosen": -105.43904876708984,
-      "logps/rejected": -217.46287536621094,
-      "loss": 0.4176,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.10640807449817657,
-      "rewards/margins": 0.26060980558395386,
-      "rewards/rejected": -0.1542017161846161,
       "step": 42
     },
     {
-      "debug/policy_chosen_logits": -1.2382934093475342,
-      "debug/policy_chosen_logps": -112.60398864746094,
-      "debug/policy_rejected_logits": -1.0149949789047241,
-      "debug/policy_rejected_logps": -285.73968505859375,
-      "debug/reference_chosen_logps": -115.7171630859375,
-      "debug/reference_rejected_logps": -250.61151123046875,
       "epoch": 0.8269230769230769,
-      "grad_norm": 36.616872187513835,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2382934093475342,
-      "logits/rejected": -1.0149949789047241,
-      "logps/chosen": -112.60398864746094,
-      "logps/rejected": -285.73968505859375,
-      "loss": 0.4429,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.031131763011217117,
-      "rewards/margins": 0.382413387298584,
-      "rewards/rejected": -0.35128161311149597,
       "step": 43
     },
     {
-      "debug/policy_chosen_logits": -1.4913691282272339,
-      "debug/policy_chosen_logps": -89.32835388183594,
-      "debug/policy_rejected_logits": -1.456650972366333,
-      "debug/policy_rejected_logps": -236.55682373046875,
-      "debug/reference_chosen_logps": -100.08968353271484,
-      "debug/reference_rejected_logps": -225.67787170410156,
       "epoch": 0.8461538461538461,
-      "grad_norm": 15.043363687225893,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.4913691282272339,
-      "logits/rejected": -1.456650972366333,
-      "logps/chosen": -89.32835388183594,
-      "logps/rejected": -236.55682373046875,
-      "loss": 0.3994,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.10761332511901855,
-      "rewards/margins": 0.21640273928642273,
-      "rewards/rejected": -0.10878939926624298,
       "step": 44
     },
     {
-      "debug/policy_chosen_logits": -1.3964580297470093,
-      "debug/policy_chosen_logps": -127.14812469482422,
-      "debug/policy_rejected_logits": -1.4482215642929077,
-      "debug/policy_rejected_logps": -165.46240234375,
-      "debug/reference_chosen_logps": -140.458984375,
-      "debug/reference_rejected_logps": -171.06005859375,
       "epoch": 0.8653846153846154,
-      "grad_norm": 49.59204781141965,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.3964580297470093,
-      "logits/rejected": -1.4482215642929077,
-      "logps/chosen": -127.14812469482422,
-      "logps/rejected": -165.46240234375,
-      "loss": 0.4419,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.13310852646827698,
-      "rewards/margins": 0.07713213562965393,
-      "rewards/rejected": 0.05597639083862305,
       "step": 45
     },
     {
-      "debug/policy_chosen_logits": -1.2927504777908325,
-      "debug/policy_chosen_logps": -114.67044830322266,
-      "debug/policy_rejected_logits": -1.279125452041626,
-      "debug/policy_rejected_logps": -271.4466247558594,
-      "debug/reference_chosen_logps": -128.33384704589844,
-      "debug/reference_rejected_logps": -258.589599609375,
       "epoch": 0.8846153846153846,
-      "grad_norm": 20.796125210807904,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2927504777908325,
-      "logits/rejected": -1.279125452041626,
-      "logps/chosen": -114.67044830322266,
-      "logps/rejected": -271.4466247558594,
-      "loss": 0.4084,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 0.136633962392807,
-      "rewards/margins": 0.2652040123939514,
-      "rewards/rejected": -0.1285700500011444,
       "step": 46
     },
     {
-      "debug/policy_chosen_logits": -1.203310251235962,
-      "debug/policy_chosen_logps": -116.76630401611328,
-      "debug/policy_rejected_logits": -1.2802678346633911,
-      "debug/policy_rejected_logps": -212.71511840820312,
-      "debug/reference_chosen_logps": -124.21476745605469,
-      "debug/reference_rejected_logps": -208.848388671875,
       "epoch": 0.9038461538461539,
-      "grad_norm": 22.549023272823217,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.203310251235962,
-      "logits/rejected": -1.2802678346633911,
-      "logps/chosen": -116.76630401611328,
-      "logps/rejected": -212.71511840820312,
-      "loss": 0.4288,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.07448464632034302,
-      "rewards/margins": 0.11315208673477173,
-      "rewards/rejected": -0.03866744041442871,
       "step": 47
     },
     {
-      "debug/policy_chosen_logits": -1.329450011253357,
-      "debug/policy_chosen_logps": -141.68182373046875,
-      "debug/policy_rejected_logits": -1.164079189300537,
-      "debug/policy_rejected_logps": -292.22705078125,
-      "debug/reference_chosen_logps": -148.44650268554688,
-      "debug/reference_rejected_logps": -282.9841003417969,
       "epoch": 0.9230769230769231,
-      "grad_norm": 38.21519514395376,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.329450011253357,
-      "logits/rejected": -1.164079189300537,
-      "logps/chosen": -141.68182373046875,
-      "logps/rejected": -292.22705078125,
-      "loss": 0.4126,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.06764666736125946,
-      "rewards/margins": 0.16007646918296814,
-      "rewards/rejected": -0.09242980927228928,
       "step": 48
     },
     {
-      "debug/policy_chosen_logits": -1.1785225868225098,
-      "debug/policy_chosen_logps": -123.70880889892578,
-      "debug/policy_rejected_logits": -1.090317964553833,
-      "debug/policy_rejected_logps": -169.21560668945312,
-      "debug/reference_chosen_logps": -136.04171752929688,
-      "debug/reference_rejected_logps": -169.77383422851562,
       "epoch": 0.9423076923076923,
-      "grad_norm": 35.07581270707382,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.1785225868225098,
-      "logits/rejected": -1.090317964553833,
-      "logps/chosen": -123.70880889892578,
-      "logps/rejected": -169.21560668945312,
-      "loss": 0.4397,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.12332899868488312,
-      "rewards/margins": 0.11774662882089615,
-      "rewards/rejected": 0.005582377314567566,
       "step": 49
     },
     {
-      "debug/policy_chosen_logits": -1.2230808734893799,
-      "debug/policy_chosen_logps": -157.45596313476562,
-      "debug/policy_rejected_logits": -1.170206904411316,
-      "debug/policy_rejected_logps": -291.5883483886719,
-      "debug/reference_chosen_logps": -161.05508422851562,
-      "debug/reference_rejected_logps": -282.8891296386719,
       "epoch": 0.9615384615384616,
-      "grad_norm": 15.529898113956751,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2230808734893799,
-      "logits/rejected": -1.170206904411316,
-      "logps/chosen": -157.45596313476562,
-      "logps/rejected": -291.5883483886719,
-      "loss": 0.3907,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.03599133342504501,
-      "rewards/margins": 0.12298347800970078,
-      "rewards/rejected": -0.08699213713407516,
       "step": 50
     },
     {
-      "debug/policy_chosen_logits": -1.3883893489837646,
-      "debug/policy_chosen_logps": -146.48504638671875,
-      "debug/policy_rejected_logits": -1.2246520519256592,
-      "debug/policy_rejected_logps": -147.98838806152344,
-      "debug/reference_chosen_logps": -149.21646118164062,
-      "debug/reference_rejected_logps": -146.49032592773438,
       "epoch": 0.9807692307692307,
-      "grad_norm": 36.66752216762139,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.3883893489837646,
-      "logits/rejected": -1.2246520519256592,
-      "logps/chosen": -146.48504638671875,
-      "logps/rejected": -147.98838806152344,
-      "loss": 0.4076,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.02731417492032051,
-      "rewards/margins": 0.04229472205042839,
-      "rewards/rejected": -0.01498054713010788,
       "step": 51
     },
     {
-      "debug/policy_chosen_logits": -1.2958321571350098,
-      "debug/policy_chosen_logps": -112.06967163085938,
-      "debug/policy_rejected_logits": -1.363411545753479,
-      "debug/policy_rejected_logps": -210.20330810546875,
-      "debug/reference_chosen_logps": -118.75313568115234,
-      "debug/reference_rejected_logps": -207.29669189453125,
       "epoch": 1.0,
-      "grad_norm": 25.501005146613775,
       "learning_rate": 1e-06,
-      "logits/chosen": -1.2958321571350098,
-      "logits/rejected": -1.363411545753479,
-      "logps/chosen": -112.06967163085938,
-      "logps/rejected": -210.20330810546875,
-      "loss": 0.3806,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.06683465838432312,
-      "rewards/margins": 0.09590078890323639,
-      "rewards/rejected": -0.02906613051891327,
       "step": 52
     },
     {
       "epoch": 1.0,
       "step": 52,
       "total_flos": 0.0,
-      "train_loss": 0.44354424568322987,
-      "train_runtime": 174.5843,
-      "train_samples_per_second": 18.742,
-      "train_steps_per_second": 0.298
     }
   ],
   "logging_steps": 1,

   "is_world_process_zero": true,
   "log_history": [
     {
+      "debug/policy_chosen_logits": -1.2348219156265259,
+      "debug/policy_chosen_logps": -132.92245483398438,
+      "debug/policy_rejected_logits": -1.1747448444366455,
+      "debug/policy_rejected_logps": -163.80661010742188,
+      "debug/reference_chosen_logps": -132.92245483398438,
+      "debug/reference_rejected_logps": -163.80661010742188,
       "epoch": 0.019230769230769232,
+      "grad_norm": 21.739564681695516,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2348219156265259,
+      "logits/rejected": -1.1747448444366455,
+      "logps/chosen": -132.92245483398438,
+      "logps/rejected": -163.80661010742188,
       "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "step": 1
     },
     {
+      "debug/policy_chosen_logits": -1.2617549896240234,
+      "debug/policy_chosen_logps": -182.63986206054688,
+      "debug/policy_rejected_logits": -1.2036871910095215,
+      "debug/policy_rejected_logps": -207.30313110351562,
+      "debug/reference_chosen_logps": -180.3394012451172,
+      "debug/reference_rejected_logps": -207.17822265625,
       "epoch": 0.038461538461538464,
+      "grad_norm": 38.38047434038335,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2617549896240234,
+      "logits/rejected": -1.2036871910095215,
+      "logps/chosen": -182.63986206054688,
+      "logps/rejected": -207.30313110351562,
+      "loss": 0.4956,
       "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.02300471067428589,
+      "rewards/margins": -0.021755550056695938,
+      "rewards/rejected": -0.0012491607340052724,
       "step": 2
     },
     {
+      "debug/policy_chosen_logits": -1.2853220701217651,
+      "debug/policy_chosen_logps": -163.3322296142578,
+      "debug/policy_rejected_logits": -1.2501897811889648,
+      "debug/policy_rejected_logps": -241.1845703125,
+      "debug/reference_chosen_logps": -157.95523071289062,
+      "debug/reference_rejected_logps": -234.6351776123047,
       "epoch": 0.057692307692307696,
+      "grad_norm": 71.93148801287602,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2853220701217651,
+      "logits/rejected": -1.2501897811889648,
+      "logps/chosen": -163.3322296142578,
+      "logps/rejected": -241.1845703125,
+      "loss": 0.5122,
       "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05377005785703659,
+      "rewards/margins": 0.011723717674612999,
+      "rewards/rejected": -0.06549377739429474,
       "step": 3
     },
     {
+      "debug/policy_chosen_logits": -1.1130611896514893,
+      "debug/policy_chosen_logps": -97.92842102050781,
+      "debug/policy_rejected_logits": -1.056685209274292,
+      "debug/policy_rejected_logps": -202.48904418945312,
+      "debug/reference_chosen_logps": -96.54649353027344,
+      "debug/reference_rejected_logps": -196.40328979492188,
       "epoch": 0.07692307692307693,
+      "grad_norm": 94.04471405781264,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1130611896514893,
+      "logits/rejected": -1.056685209274292,
+      "logps/chosen": -97.92842102050781,
+      "logps/rejected": -202.48904418945312,
+      "loss": 0.5269,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.013819340616464615,
+      "rewards/margins": 0.047038186341524124,
+      "rewards/rejected": -0.06085752695798874,
       "step": 4
     },
     {
+      "debug/policy_chosen_logits": -1.1152414083480835,
+      "debug/policy_chosen_logps": -123.81487274169922,
+      "debug/policy_rejected_logits": -1.1132621765136719,
+      "debug/policy_rejected_logps": -219.38116455078125,
+      "debug/reference_chosen_logps": -121.18955993652344,
+      "debug/reference_rejected_logps": -214.46588134765625,
       "epoch": 0.09615384615384616,
+      "grad_norm": 74.87758823721254,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1152414083480835,
+      "logits/rejected": -1.1132621765136719,
+      "logps/chosen": -123.81487274169922,
+      "logps/rejected": -219.38116455078125,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.02625308930873871,
+      "rewards/margins": 0.022899730131030083,
+      "rewards/rejected": -0.04915282130241394,
       "step": 5
     },
     {
+      "debug/policy_chosen_logits": -1.3609522581100464,
+      "debug/policy_chosen_logps": -170.71603393554688,
+      "debug/policy_rejected_logits": -1.2694591283798218,
+      "debug/policy_rejected_logps": -202.8367156982422,
+      "debug/reference_chosen_logps": -169.4237823486328,
+      "debug/reference_rejected_logps": -198.68704223632812,
       "epoch": 0.11538461538461539,
+      "grad_norm": 19.390219879650054,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3609522581100464,
+      "logits/rejected": -1.2694591283798218,
+      "logps/chosen": -170.71603393554688,
+      "logps/rejected": -202.8367156982422,
+      "loss": 0.4924,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.012922583147883415,
+      "rewards/margins": 0.028574256226420403,
+      "rewards/rejected": -0.04149683937430382,
       "step": 6
     },
     {
+      "debug/policy_chosen_logits": -1.247850775718689,
+      "debug/policy_chosen_logps": -137.57997131347656,
+      "debug/policy_rejected_logits": -1.2975486516952515,
+      "debug/policy_rejected_logps": -177.66110229492188,
+      "debug/reference_chosen_logps": -139.8856658935547,
+      "debug/reference_rejected_logps": -178.00439453125,
       "epoch": 0.1346153846153846,
+      "grad_norm": 26.09557066418132,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.247850775718689,
+      "logits/rejected": -1.2975486516952515,
+      "logps/chosen": -137.57997131347656,
+      "logps/rejected": -177.66110229492188,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.023056859150528908,
+      "rewards/margins": 0.019623905420303345,
+      "rewards/rejected": 0.003432951867580414,
       "step": 7
     },
     {
+      "debug/policy_chosen_logits": -1.1798572540283203,
+      "debug/policy_chosen_logps": -131.80767822265625,
+      "debug/policy_rejected_logits": -1.2539470195770264,
+      "debug/policy_rejected_logps": -215.11492919921875,
+      "debug/reference_chosen_logps": -136.596435546875,
+      "debug/reference_rejected_logps": -221.34326171875,
       "epoch": 0.15384615384615385,
+      "grad_norm": 47.26951009616528,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1798572540283203,
+      "logits/rejected": -1.2539470195770264,
+      "logps/chosen": -131.80767822265625,
+      "logps/rejected": -215.11492919921875,
+      "loss": 0.4814,
       "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.047887563705444336,
+      "rewards/margins": -0.014395834878087044,
+      "rewards/rejected": 0.06228340044617653,
       "step": 8
     },
     {
+      "debug/policy_chosen_logits": -1.153782844543457,
+      "debug/policy_chosen_logps": -130.36990356445312,
+      "debug/policy_rejected_logits": -1.068179965019226,
+      "debug/policy_rejected_logps": -165.0654754638672,
+      "debug/reference_chosen_logps": -133.1195068359375,
+      "debug/reference_rejected_logps": -170.12216186523438,
       "epoch": 0.17307692307692307,
+      "grad_norm": 71.68724756772349,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.153782844543457,
+      "logits/rejected": -1.068179965019226,
+      "logps/chosen": -130.36990356445312,
+      "logps/rejected": -165.0654754638672,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.027496041730046272,
+      "rewards/margins": -0.02307066321372986,
+      "rewards/rejected": 0.05056670308113098,
       "step": 9
     },
     {
+      "debug/policy_chosen_logits": -1.2667584419250488,
+      "debug/policy_chosen_logps": -111.53315734863281,
+      "debug/policy_rejected_logits": -1.161044955253601,
+      "debug/policy_rejected_logps": -214.258056640625,
+      "debug/reference_chosen_logps": -117.96336364746094,
+      "debug/reference_rejected_logps": -211.9200439453125,
       "epoch": 0.19230769230769232,
+      "grad_norm": 31.608202127314065,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2667584419250488,
+      "logits/rejected": -1.161044955253601,
+      "logps/chosen": -111.53315734863281,
+      "logps/rejected": -214.258056640625,
+      "loss": 0.4554,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06430201232433319,
+      "rewards/margins": 0.08768201619386673,
+      "rewards/rejected": -0.02338000386953354,
       "step": 10
     },
     {
+      "debug/policy_chosen_logits": -1.2289499044418335,
+      "debug/policy_chosen_logps": -148.73593139648438,
+      "debug/policy_rejected_logits": -1.1797515153884888,
+      "debug/policy_rejected_logps": -285.1352844238281,
+      "debug/reference_chosen_logps": -146.48251342773438,
+      "debug/reference_rejected_logps": -279.6494140625,
       "epoch": 0.21153846153846154,
+      "grad_norm": 16.687527654352305,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2289499044418335,
+      "logits/rejected": -1.1797515153884888,
+      "logps/chosen": -148.73593139648438,
+      "logps/rejected": -285.1352844238281,
+      "loss": 0.4788,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.022534340620040894,
+      "rewards/margins": 0.03232429176568985,
+      "rewards/rejected": -0.05485863611102104,
       "step": 11
     },
     {
+      "debug/policy_chosen_logits": -0.9578278064727783,
+      "debug/policy_chosen_logps": -136.0295867919922,
+      "debug/policy_rejected_logits": -1.1070349216461182,
+      "debug/policy_rejected_logps": -274.4041748046875,
+      "debug/reference_chosen_logps": -139.7628173828125,
+      "debug/reference_rejected_logps": -272.2436828613281,
       "epoch": 0.23076923076923078,
+      "grad_norm": 23.74518292076916,
       "learning_rate": 1e-06,
+      "logits/chosen": -0.9578278064727783,
+      "logits/rejected": -1.1070349216461182,
+      "logps/chosen": -136.0295867919922,
+      "logps/rejected": -274.4041748046875,
+      "loss": 0.4588,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03733229637145996,
+      "rewards/margins": 0.05893722176551819,
+      "rewards/rejected": -0.021604929119348526,
       "step": 12
     },
     {
+      "debug/policy_chosen_logits": -1.1884337663650513,
+      "debug/policy_chosen_logps": -138.00079345703125,
+      "debug/policy_rejected_logits": -1.2273510694503784,
+      "debug/policy_rejected_logps": -236.30642700195312,
+      "debug/reference_chosen_logps": -140.82666015625,
+      "debug/reference_rejected_logps": -231.79005432128906,
       "epoch": 0.25,
+      "grad_norm": 52.574191625685216,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1884337663650513,
+      "logits/rejected": -1.2273510694503784,
+      "logps/chosen": -138.00079345703125,
+      "logps/rejected": -236.30642700195312,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0282585509121418,
+      "rewards/margins": 0.07342230528593063,
+      "rewards/rejected": -0.04516375809907913,
       "step": 13
     },
     {
+      "debug/policy_chosen_logits": -1.2762030363082886,
+      "debug/policy_chosen_logps": -109.97171020507812,
+      "debug/policy_rejected_logits": -1.2517166137695312,
+      "debug/policy_rejected_logps": -240.42059326171875,
+      "debug/reference_chosen_logps": -110.38009643554688,
+      "debug/reference_rejected_logps": -237.3839111328125,
       "epoch": 0.2692307692307692,
+      "grad_norm": 79.48353423155464,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2762030363082886,
+      "logits/rejected": -1.2517166137695312,
+      "logps/chosen": -109.97171020507812,
+      "logps/rejected": -240.42059326171875,
+      "loss": 0.4585,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.004083747044205666,
+      "rewards/margins": 0.03445054590702057,
+      "rewards/rejected": -0.030366800725460052,
       "step": 14
     },
     {
+      "debug/policy_chosen_logits": -1.258913516998291,
+      "debug/policy_chosen_logps": -119.00064086914062,
+      "debug/policy_rejected_logits": -1.3117754459381104,
+      "debug/policy_rejected_logps": -206.278076171875,
+      "debug/reference_chosen_logps": -109.17572021484375,
+      "debug/reference_rejected_logps": -200.13653564453125,
       "epoch": 0.28846153846153844,
+      "grad_norm": 112.2398036104226,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.258913516998291,
+      "logits/rejected": -1.3117754459381104,
+      "logps/chosen": -119.00064086914062,
+      "logps/rejected": -206.278076171875,
+      "loss": 0.5113,
       "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.098249152302742,
+      "rewards/margins": -0.036833763122558594,
+      "rewards/rejected": -0.06141539663076401,
       "step": 15
     },
     {
+      "debug/policy_chosen_logits": -1.2379214763641357,
+      "debug/policy_chosen_logps": -154.35321044921875,
+      "debug/policy_rejected_logits": -1.1635435819625854,
+      "debug/policy_rejected_logps": -249.5124969482422,
+      "debug/reference_chosen_logps": -147.48687744140625,
+      "debug/reference_rejected_logps": -231.68386840820312,
       "epoch": 0.3076923076923077,
+      "grad_norm": 124.12252031892565,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2379214763641357,
+      "logits/rejected": -1.1635435819625854,
+      "logps/chosen": -154.35321044921875,
+      "logps/rejected": -249.5124969482422,
+      "loss": 0.4921,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.06866332143545151,
+      "rewards/margins": 0.1096230149269104,
+      "rewards/rejected": -0.1782863438129425,
       "step": 16
     },
     {
+      "debug/policy_chosen_logits": -1.2502650022506714,
+      "debug/policy_chosen_logps": -152.2196807861328,
+      "debug/policy_rejected_logits": -1.1885775327682495,
+      "debug/policy_rejected_logps": -176.0025177001953,
+      "debug/reference_chosen_logps": -151.514892578125,
+      "debug/reference_rejected_logps": -162.71047973632812,
       "epoch": 0.3269230769230769,
+      "grad_norm": 91.49358092625756,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2502650022506714,
+      "logits/rejected": -1.1885775327682495,
+      "logps/chosen": -152.2196807861328,
+      "logps/rejected": -176.0025177001953,
+      "loss": 0.4641,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007047949358820915,
+      "rewards/margins": 0.12587252259254456,
+      "rewards/rejected": -0.13292045891284943,
       "step": 17
     },
     {
+      "debug/policy_chosen_logits": -1.2029650211334229,
+      "debug/policy_chosen_logps": -168.9429473876953,
+      "debug/policy_rejected_logits": -1.211535930633545,
+      "debug/policy_rejected_logps": -307.27362060546875,
+      "debug/reference_chosen_logps": -166.69866943359375,
+      "debug/reference_rejected_logps": -267.7721252441406,
       "epoch": 0.34615384615384615,
+      "grad_norm": 30.34097826458164,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2029650211334229,
+      "logits/rejected": -1.211535930633545,
+      "logps/chosen": -168.9429473876953,
+      "logps/rejected": -307.27362060546875,
+      "loss": 0.4407,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.02244272269308567,
+      "rewards/margins": 0.37257200479507446,
+      "rewards/rejected": -0.39501476287841797,
       "step": 18
     },
     {
+      "debug/policy_chosen_logits": -1.3290777206420898,
+      "debug/policy_chosen_logps": -113.09046936035156,
+      "debug/policy_rejected_logits": -1.2192254066467285,
+      "debug/policy_rejected_logps": -211.23617553710938,
+      "debug/reference_chosen_logps": -116.71571350097656,
+      "debug/reference_rejected_logps": -208.08888244628906,
       "epoch": 0.36538461538461536,
+      "grad_norm": 15.641150451814267,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3290777206420898,
+      "logits/rejected": -1.2192254066467285,
+      "logps/chosen": -113.09046936035156,
+      "logps/rejected": -211.23617553710938,
+      "loss": 0.4401,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.036252379417419434,
+      "rewards/margins": 0.06772524118423462,
+      "rewards/rejected": -0.031472865492105484,
       "step": 19
     },
     {
+      "debug/policy_chosen_logits": -1.3350298404693604,
+      "debug/policy_chosen_logps": -139.8019561767578,
+      "debug/policy_rejected_logits": -1.3234355449676514,
+      "debug/policy_rejected_logps": -252.89657592773438,
+      "debug/reference_chosen_logps": -143.83834838867188,
+      "debug/reference_rejected_logps": -245.260986328125,
       "epoch": 0.38461538461538464,
+      "grad_norm": 26.92984184652646,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3350298404693604,
+      "logits/rejected": -1.3234355449676514,
+      "logps/chosen": -139.8019561767578,
+      "logps/rejected": -252.89657592773438,
+      "loss": 0.4678,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04036390781402588,
+      "rewards/margins": 0.11671990901231766,
+      "rewards/rejected": -0.07635599374771118,
       "step": 20
     },
     {
+      "debug/policy_chosen_logits": -1.2050725221633911,
+      "debug/policy_chosen_logps": -141.1668243408203,
+      "debug/policy_rejected_logits": -1.222265601158142,
+      "debug/policy_rejected_logps": -172.55345153808594,
+      "debug/reference_chosen_logps": -141.07467651367188,
+      "debug/reference_rejected_logps": -171.04931640625,
       "epoch": 0.40384615384615385,
+      "grad_norm": 53.89373486771968,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2050725221633911,
+      "logits/rejected": -1.222265601158142,
+      "logps/chosen": -141.1668243408203,
+      "logps/rejected": -172.55345153808594,
+      "loss": 0.4892,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0009214691817760468,
+      "rewards/margins": 0.014119969680905342,
+      "rewards/rejected": -0.01504143700003624,
       "step": 21
     },
     {
+      "debug/policy_chosen_logits": -1.2559270858764648,
+      "debug/policy_chosen_logps": -154.6681365966797,
+      "debug/policy_rejected_logits": -1.1459161043167114,
+      "debug/policy_rejected_logps": -176.13983154296875,
+      "debug/reference_chosen_logps": -158.22708129882812,
+      "debug/reference_rejected_logps": -175.68319702148438,
       "epoch": 0.4230769230769231,
+      "grad_norm": 20.393196406563664,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2559270858764648,
+      "logits/rejected": -1.1459161043167114,
+      "logps/chosen": -154.6681365966797,
+      "logps/rejected": -176.13983154296875,
+      "loss": 0.4368,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03558942675590515,
+      "rewards/margins": 0.04015577584505081,
+      "rewards/rejected": -0.004566345363855362,
       "step": 22
     },
     {
+      "debug/policy_chosen_logits": -1.099912166595459,
+      "debug/policy_chosen_logps": -99.30662536621094,
+      "debug/policy_rejected_logits": -1.038980484008789,
+      "debug/policy_rejected_logps": -330.9873046875,
+      "debug/reference_chosen_logps": -104.75813293457031,
+      "debug/reference_rejected_logps": -311.56396484375,
       "epoch": 0.4423076923076923,
+      "grad_norm": 15.504920171360686,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.099912166595459,
+      "logits/rejected": -1.038980484008789,
+      "logps/chosen": -99.30662536621094,
+      "logps/rejected": -330.9873046875,
+      "loss": 0.4496,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05451509356498718,
+      "rewards/margins": 0.24874866008758545,
+      "rewards/rejected": -0.19423356652259827,
       "step": 23
     },
     {
+      "debug/policy_chosen_logits": -1.353391170501709,
+      "debug/policy_chosen_logps": -152.0869598388672,
+      "debug/policy_rejected_logits": -1.306003451347351,
+      "debug/policy_rejected_logps": -324.70758056640625,
+      "debug/reference_chosen_logps": -152.76806640625,
+      "debug/reference_rejected_logps": -304.7029724121094,
       "epoch": 0.46153846153846156,
+      "grad_norm": 37.30857181130373,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.353391170501709,
+      "logits/rejected": -1.306003451347351,
+      "logps/chosen": -152.0869598388672,
+      "logps/rejected": -324.70758056640625,
+      "loss": 0.4288,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.006811168976128101,
+      "rewards/margins": 0.2068571001291275,
+      "rewards/rejected": -0.20004592835903168,
       "step": 24
     },
     {
+      "debug/policy_chosen_logits": -1.2506132125854492,
+      "debug/policy_chosen_logps": -182.32362365722656,
+      "debug/policy_rejected_logits": -1.2176698446273804,
+      "debug/policy_rejected_logps": -295.3228454589844,
+      "debug/reference_chosen_logps": -182.55697631835938,
+      "debug/reference_rejected_logps": -282.0664978027344,
       "epoch": 0.4807692307692308,
+      "grad_norm": 57.662326320570806,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2506132125854492,
+      "logits/rejected": -1.2176698446273804,
+      "logps/chosen": -182.32362365722656,
+      "logps/rejected": -295.3228454589844,
+      "loss": 0.4189,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.002333402633666992,
+      "rewards/margins": 0.13489675521850586,
+      "rewards/rejected": -0.13256335258483887,
       "step": 25
     },
     {
+      "debug/policy_chosen_logits": -1.2043673992156982,
+      "debug/policy_chosen_logps": -139.78085327148438,
+      "debug/policy_rejected_logits": -1.3776921033859253,
+      "debug/policy_rejected_logps": -216.3494873046875,
+      "debug/reference_chosen_logps": -146.04248046875,
+      "debug/reference_rejected_logps": -214.1402587890625,
       "epoch": 0.5,
+      "grad_norm": 22.330670420413643,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2043673992156982,
+      "logits/rejected": -1.3776921033859253,
+      "logps/chosen": -139.78085327148438,
+      "logps/rejected": -216.3494873046875,
+      "loss": 0.4117,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06261642277240753,
+      "rewards/margins": 0.0847088098526001,
+      "rewards/rejected": -0.02209237962961197,
       "step": 26
     },
     {
+      "debug/policy_chosen_logits": -1.211961030960083,
+      "debug/policy_chosen_logps": -144.8836669921875,
+      "debug/policy_rejected_logits": -1.2210055589675903,
+      "debug/policy_rejected_logps": -174.15650939941406,
+      "debug/reference_chosen_logps": -143.75244140625,
+      "debug/reference_rejected_logps": -176.13311767578125,
       "epoch": 0.5192307692307693,
+      "grad_norm": 43.60544149312096,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.211961030960083,
+      "logits/rejected": -1.2210055589675903,
+      "logps/chosen": -144.8836669921875,
+      "logps/rejected": -174.15650939941406,
+      "loss": 0.4296,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.011312179267406464,
+      "rewards/margins": -0.031078338623046875,
+      "rewards/rejected": 0.01976615935564041,
       "step": 27
     },
     {
+      "debug/policy_chosen_logits": -1.4086847305297852,
+      "debug/policy_chosen_logps": -136.67764282226562,
+      "debug/policy_rejected_logits": -1.4001612663269043,
+      "debug/policy_rejected_logps": -173.57293701171875,
+      "debug/reference_chosen_logps": -139.71961975097656,
+      "debug/reference_rejected_logps": -167.02276611328125,
       "epoch": 0.5384615384615384,
+      "grad_norm": 41.878595172037,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.4086847305297852,
+      "logits/rejected": -1.4001612663269043,
+      "logps/chosen": -136.67764282226562,
+      "logps/rejected": -173.57293701171875,
+      "loss": 0.437,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.030419737100601196,
+      "rewards/margins": 0.09592136740684509,
+      "rewards/rejected": -0.0655016228556633,
       "step": 28
     },
     {
+      "debug/policy_chosen_logits": -1.394635796546936,
+      "debug/policy_chosen_logps": -155.0882568359375,
+      "debug/policy_rejected_logits": -1.1727089881896973,
+      "debug/policy_rejected_logps": -295.13409423828125,
+      "debug/reference_chosen_logps": -160.80213928222656,
+      "debug/reference_rejected_logps": -277.72723388671875,
       "epoch": 0.5576923076923077,
+      "grad_norm": 25.614929675079054,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.394635796546936,
+      "logits/rejected": -1.1727089881896973,
+      "logps/chosen": -155.0882568359375,
+      "logps/rejected": -295.13409423828125,
+      "loss": 0.4438,
       "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.057138726115226746,
+      "rewards/margins": 0.2312072068452835,
+      "rewards/rejected": -0.17406848073005676,
       "step": 29
     },
     {
+      "debug/policy_chosen_logits": -1.3357499837875366,
+      "debug/policy_chosen_logps": -135.82276916503906,
+      "debug/policy_rejected_logits": -1.2643619775772095,
+      "debug/policy_rejected_logps": -198.8779296875,
+      "debug/reference_chosen_logps": -144.52224731445312,
+      "debug/reference_rejected_logps": -193.8353271484375,
       "epoch": 0.5769230769230769,
+      "grad_norm": 35.0050669992775,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3357499837875366,
+      "logits/rejected": -1.2643619775772095,
+      "logps/chosen": -135.82276916503906,
+      "logps/rejected": -198.8779296875,
+      "loss": 0.4463,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08699464797973633,
+      "rewards/margins": 0.1374206840991974,
+      "rewards/rejected": -0.050426043570041656,
       "step": 30
     },
     {
+      "debug/policy_chosen_logits": -1.3438653945922852,
+      "debug/policy_chosen_logps": -102.52204132080078,
+      "debug/policy_rejected_logits": -1.1623094081878662,
+      "debug/policy_rejected_logps": -243.15985107421875,
+      "debug/reference_chosen_logps": -112.55838012695312,
+      "debug/reference_rejected_logps": -241.82275390625,
       "epoch": 0.5961538461538461,
+      "grad_norm": 17.73778958561842,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3438653945922852,
+      "logits/rejected": -1.1623094081878662,
+      "logps/chosen": -102.52204132080078,
+      "logps/rejected": -243.15985107421875,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.10036339610815048,
+      "rewards/margins": 0.11373443156480789,
+      "rewards/rejected": -0.013371038250625134,
       "step": 31
     },
     {
+      "debug/policy_chosen_logits": -1.2000988721847534,
+      "debug/policy_chosen_logps": -114.93376159667969,
+      "debug/policy_rejected_logits": -0.9910339117050171,
+      "debug/policy_rejected_logps": -330.4576416015625,
+      "debug/reference_chosen_logps": -141.095703125,
+      "debug/reference_rejected_logps": -320.2563171386719,
       "epoch": 0.6153846153846154,
+      "grad_norm": 43.15694706560031,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2000988721847534,
+      "logits/rejected": -0.9910339117050171,
+      "logps/chosen": -114.93376159667969,
+      "logps/rejected": -330.4576416015625,
+      "loss": 0.4226,
       "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2616194188594818,
+      "rewards/margins": 0.3636327385902405,
+      "rewards/rejected": -0.10201331228017807,
       "step": 32
     },
     {
+      "debug/policy_chosen_logits": -1.1911633014678955,
+      "debug/policy_chosen_logps": -96.82029724121094,
+      "debug/policy_rejected_logits": -1.2555269002914429,
+      "debug/policy_rejected_logps": -276.76422119140625,
+      "debug/reference_chosen_logps": -113.63371276855469,
+      "debug/reference_rejected_logps": -275.29156494140625,
       "epoch": 0.6346153846153846,
+      "grad_norm": 44.04898042405896,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1911633014678955,
+      "logits/rejected": -1.2555269002914429,
+      "logps/chosen": -96.82029724121094,
+      "logps/rejected": -276.76422119140625,
+      "loss": 0.3996,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.16813413798809052,
+      "rewards/margins": 0.18286080658435822,
+      "rewards/rejected": -0.014726676046848297,
       "step": 33
     },
     {
+      "debug/policy_chosen_logits": -1.3251279592514038,
+      "debug/policy_chosen_logps": -133.66395568847656,
+      "debug/policy_rejected_logits": -1.258199691772461,
+      "debug/policy_rejected_logps": -242.9014892578125,
+      "debug/reference_chosen_logps": -143.2382049560547,
+      "debug/reference_rejected_logps": -244.9378662109375,
       "epoch": 0.6538461538461539,
+      "grad_norm": 115.38013119786002,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3251279592514038,
+      "logits/rejected": -1.258199691772461,
+      "logps/chosen": -133.66395568847656,
+      "logps/rejected": -242.9014892578125,
+      "loss": 0.4549,
       "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0957425907254219,
+      "rewards/margins": 0.07537883520126343,
+      "rewards/rejected": 0.02036374993622303,
       "step": 34
     },
     {
+      "debug/policy_chosen_logits": -1.3226226568222046,
+      "debug/policy_chosen_logps": -112.6985855102539,
+      "debug/policy_rejected_logits": -1.1881129741668701,
+      "debug/policy_rejected_logps": -230.60443115234375,
+      "debug/reference_chosen_logps": -130.98614501953125,
+      "debug/reference_rejected_logps": -228.2262725830078,
       "epoch": 0.6730769230769231,
+      "grad_norm": 40.4402369628288,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3226226568222046,
+      "logits/rejected": -1.1881129741668701,
+      "logps/chosen": -112.6985855102539,
+      "logps/rejected": -230.60443115234375,
+      "loss": 0.4216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18287554383277893,
+      "rewards/margins": 0.20665717124938965,
+      "rewards/rejected": -0.023781631141901016,
       "step": 35
     },
     {
+      "debug/policy_chosen_logits": -1.1547788381576538,
+      "debug/policy_chosen_logps": -118.73101806640625,
+      "debug/policy_rejected_logits": -1.0697778463363647,
+      "debug/policy_rejected_logps": -324.0631103515625,
+      "debug/reference_chosen_logps": -127.16613006591797,
+      "debug/reference_rejected_logps": -306.434326171875,
       "epoch": 0.6923076923076923,
+      "grad_norm": 18.889421126870324,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1547788381576538,
+      "logits/rejected": -1.0697778463363647,
+      "logps/chosen": -118.73101806640625,
+      "logps/rejected": -324.0631103515625,
+      "loss": 0.4124,
       "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08435116708278656,
+      "rewards/margins": 0.2606390118598938,
+      "rewards/rejected": -0.17628784477710724,
       "step": 36
     },
     {
+      "debug/policy_chosen_logits": -1.2630081176757812,
+      "debug/policy_chosen_logps": -141.00460815429688,
+      "debug/policy_rejected_logits": -1.2191274166107178,
+      "debug/policy_rejected_logps": -196.04946899414062,
+      "debug/reference_chosen_logps": -141.60374450683594,
+      "debug/reference_rejected_logps": -181.31646728515625,
       "epoch": 0.7115384615384616,
+      "grad_norm": 29.84262997736804,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2630081176757812,
+      "logits/rejected": -1.2191274166107178,
+      "logps/chosen": -141.00460815429688,
+      "logps/rejected": -196.04946899414062,
+      "loss": 0.4429,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0059913452714681625,
+      "rewards/margins": 0.15332148969173431,
+      "rewards/rejected": -0.1473301351070404,
       "step": 37
     },
     {
+      "debug/policy_chosen_logits": -1.4539029598236084,
+      "debug/policy_chosen_logps": -123.56523895263672,
+      "debug/policy_rejected_logits": -1.4236867427825928,
+      "debug/policy_rejected_logps": -185.32632446289062,
+      "debug/reference_chosen_logps": -129.4361114501953,
+      "debug/reference_rejected_logps": -179.44058227539062,
       "epoch": 0.7307692307692307,
+      "grad_norm": 17.23655033742978,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.4539029598236084,
+      "logits/rejected": -1.4236867427825928,
+      "logps/chosen": -123.56523895263672,
+      "logps/rejected": -185.32632446289062,
+      "loss": 0.3743,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05870867520570755,
+      "rewards/margins": 0.11756613105535507,
+      "rewards/rejected": -0.05885745957493782,
       "step": 38
     },
     {
+      "debug/policy_chosen_logits": -1.248382806777954,
+      "debug/policy_chosen_logps": -220.80645751953125,
+      "debug/policy_rejected_logits": -1.1765296459197998,
+      "debug/policy_rejected_logps": -198.0468292236328,
+      "debug/reference_chosen_logps": -208.3310546875,
+      "debug/reference_rejected_logps": -192.87469482421875,
       "epoch": 0.75,
+      "grad_norm": 42.21364154787503,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.248382806777954,
+      "logits/rejected": -1.1765296459197998,
+      "logps/chosen": -220.80645751953125,
+      "logps/rejected": -198.0468292236328,
+      "loss": 0.456,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.12475401908159256,
+      "rewards/margins": -0.07303276658058167,
+      "rewards/rejected": -0.051721252501010895,
       "step": 39
     },
     {
+      "debug/policy_chosen_logits": -1.420503854751587,
+      "debug/policy_chosen_logps": -150.28591918945312,
+      "debug/policy_rejected_logits": -1.3057647943496704,
+      "debug/policy_rejected_logps": -263.63275146484375,
+      "debug/reference_chosen_logps": -148.029052734375,
+      "debug/reference_rejected_logps": -224.35903930664062,
       "epoch": 0.7692307692307693,
+      "grad_norm": 100.89795058616748,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.420503854751587,
+      "logits/rejected": -1.3057647943496704,
+      "logps/chosen": -150.28591918945312,
+      "logps/rejected": -263.63275146484375,
+      "loss": 0.4926,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.02256855182349682,
+      "rewards/margins": 0.3701684772968292,
+      "rewards/rejected": -0.3927370607852936,
       "step": 40
     },
     {
+      "debug/policy_chosen_logits": -1.2056798934936523,
+      "debug/policy_chosen_logps": -183.70773315429688,
+      "debug/policy_rejected_logits": -1.1638095378875732,
+      "debug/policy_rejected_logps": -303.64556884765625,
+      "debug/reference_chosen_logps": -184.8590087890625,
+      "debug/reference_rejected_logps": -268.26873779296875,
       "epoch": 0.7884615384615384,
+      "grad_norm": 87.52835684552551,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2056798934936523,
+      "logits/rejected": -1.1638095378875732,
+      "logps/chosen": -183.70773315429688,
+      "logps/rejected": -303.64556884765625,
+      "loss": 0.4298,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.011512821540236473,
+      "rewards/margins": 0.36528119444847107,
+      "rewards/rejected": -0.35376837849617004,
       "step": 41
     },
     {
+      "debug/policy_chosen_logits": -1.274660587310791,
+      "debug/policy_chosen_logps": -109.7809066772461,
+      "debug/policy_rejected_logits": -1.2929130792617798,
+      "debug/policy_rejected_logps": -221.91757202148438,
+      "debug/reference_chosen_logps": -117.5357437133789,
+      "debug/reference_rejected_logps": -203.26840209960938,
       "epoch": 0.8076923076923077,
+      "grad_norm": 133.93511541568418,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.274660587310791,
+      "logits/rejected": -1.2929130792617798,
+      "logps/chosen": -109.7809066772461,
+      "logps/rejected": -221.91757202148438,
+      "loss": 0.4672,
       "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.07754837721586227,
+      "rewards/margins": 0.26404017210006714,
+      "rewards/rejected": -0.18649178743362427,
       "step": 42
     },
     {
+      "debug/policy_chosen_logits": -1.229216456413269,
+      "debug/policy_chosen_logps": -115.21796417236328,
+      "debug/policy_rejected_logits": -1.0166616439819336,
+      "debug/policy_rejected_logps": -288.5059814453125,
+      "debug/reference_chosen_logps": -118.31993103027344,
+      "debug/reference_rejected_logps": -250.616943359375,
       "epoch": 0.8269230769230769,
+      "grad_norm": 77.32000216633205,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.229216456413269,
+      "logits/rejected": -1.0166616439819336,
+      "logps/chosen": -115.21796417236328,
+      "logps/rejected": -288.5059814453125,
+      "loss": 0.4625,
       "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.031019629910588264,
+      "rewards/margins": 0.4099102020263672,
+      "rewards/rejected": -0.3788905739784241,
       "step": 43
     },
     {
+      "debug/policy_chosen_logits": -1.4798791408538818,
+      "debug/policy_chosen_logps": -89.95073699951172,
+      "debug/policy_rejected_logits": -1.4541199207305908,
+      "debug/policy_rejected_logps": -239.7828826904297,
+      "debug/reference_chosen_logps": -101.15357971191406,
+      "debug/reference_rejected_logps": -227.21592712402344,
       "epoch": 0.8461538461538461,
+      "grad_norm": 40.276665507330875,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.4798791408538818,
+      "logits/rejected": -1.4541199207305908,
+      "logps/chosen": -89.95073699951172,
+      "logps/rejected": -239.7828826904297,
+      "loss": 0.4063,
       "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.11202842742204666,
+      "rewards/margins": 0.2376978099346161,
+      "rewards/rejected": -0.12566937506198883,
       "step": 44
     },
     {
+      "debug/policy_chosen_logits": -1.385978102684021,
+      "debug/policy_chosen_logps": -129.73675537109375,
+      "debug/policy_rejected_logits": -1.4437228441238403,
+      "debug/policy_rejected_logps": -166.5294189453125,
+      "debug/reference_chosen_logps": -142.4840850830078,
+      "debug/reference_rejected_logps": -174.26443481445312,
       "epoch": 0.8653846153846154,
+      "grad_norm": 16.81392684345585,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.385978102684021,
+      "logits/rejected": -1.4437228441238403,
+      "logps/chosen": -129.73675537109375,
+      "logps/rejected": -166.5294189453125,
+      "loss": 0.4461,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.12747323513031006,
+      "rewards/margins": 0.050123024731874466,
+      "rewards/rejected": 0.0773501992225647,
       "step": 45
     },
     {
+      "debug/policy_chosen_logits": -1.2822701930999756,
+      "debug/policy_chosen_logps": -117.16873168945312,
+      "debug/policy_rejected_logits": -1.2780365943908691,
+      "debug/policy_rejected_logps": -270.9895324707031,
+      "debug/reference_chosen_logps": -130.97329711914062,
+      "debug/reference_rejected_logps": -259.4662170410156,
       "epoch": 0.8846153846153846,
+      "grad_norm": 20.14742752061246,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2822701930999756,
+      "logits/rejected": -1.2780365943908691,
+      "logps/chosen": -117.16873168945312,
+      "logps/rejected": -270.9895324707031,
+      "loss": 0.4036,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.13804557919502258,
+      "rewards/margins": 0.2532787024974823,
+      "rewards/rejected": -0.11523310840129852,
       "step": 46
     },
     {
+      "debug/policy_chosen_logits": -1.1911503076553345,
+      "debug/policy_chosen_logps": -116.65675354003906,
+      "debug/policy_rejected_logits": -1.2724329233169556,
+      "debug/policy_rejected_logps": -212.22393798828125,
+      "debug/reference_chosen_logps": -127.14996337890625,
+      "debug/reference_rejected_logps": -210.3226776123047,
       "epoch": 0.9038461538461539,
+      "grad_norm": 36.81625781029569,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1911503076553345,
+      "logits/rejected": -1.2724329233169556,
+      "logps/chosen": -116.65675354003906,
+      "logps/rejected": -212.22393798828125,
+      "loss": 0.4372,
       "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.10493214428424835,
+      "rewards/margins": 0.12394469976425171,
+      "rewards/rejected": -0.01901254430413246,
       "step": 47
     },
     {
+      "debug/policy_chosen_logits": -1.3235502243041992,
+      "debug/policy_chosen_logps": -142.3896942138672,
+      "debug/policy_rejected_logits": -1.16089928150177,
+      "debug/policy_rejected_logps": -294.1658935546875,
+      "debug/reference_chosen_logps": -148.7461395263672,
+      "debug/reference_rejected_logps": -284.9529113769531,
       "epoch": 0.9230769230769231,
+      "grad_norm": 55.59751943354035,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3235502243041992,
+      "logits/rejected": -1.16089928150177,
+      "logps/chosen": -142.3896942138672,
+      "logps/rejected": -294.1658935546875,
+      "loss": 0.4307,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.06356436759233475,
+      "rewards/margins": 0.15569432079792023,
+      "rewards/rejected": -0.09212995320558548,
       "step": 48
     },
     {
+      "debug/policy_chosen_logits": -1.1583129167556763,
+      "debug/policy_chosen_logps": -124.10246276855469,
+      "debug/policy_rejected_logits": -1.0836573839187622,
+      "debug/policy_rejected_logps": -168.42910766601562,
+      "debug/reference_chosen_logps": -138.41485595703125,
+      "debug/reference_rejected_logps": -172.309326171875,
       "epoch": 0.9423076923076923,
+      "grad_norm": 63.48037385177918,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.1583129167556763,
+      "logits/rejected": -1.0836573839187622,
+      "logps/chosen": -124.10246276855469,
+      "logps/rejected": -168.42910766601562,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1431237757205963,
+      "rewards/margins": 0.10432147234678268,
+      "rewards/rejected": 0.038802310824394226,
       "step": 49
     },
     {
+      "debug/policy_chosen_logits": -1.2059780359268188,
+      "debug/policy_chosen_logps": -157.7839813232422,
+      "debug/policy_rejected_logits": -1.1662678718566895,
+      "debug/policy_rejected_logps": -291.1665954589844,
+      "debug/reference_chosen_logps": -163.0614013671875,
+      "debug/reference_rejected_logps": -284.3927307128906,
       "epoch": 0.9615384615384616,
+      "grad_norm": 25.37123962222348,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2059780359268188,
+      "logits/rejected": -1.1662678718566895,
+      "logps/chosen": -157.7839813232422,
+      "logps/rejected": -291.1665954589844,
+      "loss": 0.3946,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.052774280309677124,
+      "rewards/margins": 0.1205129325389862,
+      "rewards/rejected": -0.06773865222930908,
       "step": 50
     },
     {
+      "debug/policy_chosen_logits": -1.3792681694030762,
+      "debug/policy_chosen_logps": -146.66757202148438,
+      "debug/policy_rejected_logits": -1.214746117591858,
+      "debug/policy_rejected_logps": -145.07708740234375,
+      "debug/reference_chosen_logps": -151.08615112304688,
+      "debug/reference_rejected_logps": -149.86410522460938,
       "epoch": 0.9807692307692307,
+      "grad_norm": 18.070723141611428,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.3792681694030762,
+      "logits/rejected": -1.214746117591858,
+      "logps/chosen": -146.66757202148438,
+      "logps/rejected": -145.07708740234375,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.04418577253818512,
+      "rewards/margins": -0.0036845793947577477,
+      "rewards/rejected": 0.047870345413684845,
       "step": 51
     },
     {
+      "debug/policy_chosen_logits": -1.2889589071273804,
+      "debug/policy_chosen_logps": -117.15675354003906,
+      "debug/policy_rejected_logits": -1.353893518447876,
+      "debug/policy_rejected_logps": -214.70697021484375,
+      "debug/reference_chosen_logps": -121.83930969238281,
+      "debug/reference_rejected_logps": -210.53515625,
       "epoch": 1.0,
+      "grad_norm": 37.32087719041123,
       "learning_rate": 1e-06,
+      "logits/chosen": -1.2889589071273804,
+      "logits/rejected": -1.353893518447876,
+      "logps/chosen": -117.15675354003906,
+      "logps/rejected": -214.70697021484375,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0468255952000618,
+      "rewards/margins": 0.08854364603757858,
+      "rewards/rejected": -0.04171804338693619,
       "step": 52
     },
     {
       "epoch": 1.0,
       "step": 52,
       "total_flos": 0.0,
+      "train_loss": 0.45137535952604735,
+      "train_runtime": 176.3389,
+      "train_samples_per_second": 18.555,
+      "train_steps_per_second": 0.295
     }
   ],
   "logging_steps": 1,