| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 33, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": -2.4703779220581055, | |
| "debug/policy_chosen_logps": -179.5443115234375, | |
| "debug/policy_rejected_logits": -2.4724299907684326, | |
| "debug/policy_rejected_logps": -182.86181640625, | |
| "debug/reference_chosen_logps": -179.5443115234375, | |
| "debug/reference_rejected_logps": -182.86181640625, | |
| "epoch": 0.030303030303030304, | |
| "grad_norm": 2.750340847346066, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.4703779220581055, | |
| "logits/rejected": -2.4724299907684326, | |
| "logps/chosen": -179.5443115234375, | |
| "logps/rejected": -182.86181640625, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.612560987472534, | |
| "debug/policy_chosen_logps": -187.98855590820312, | |
| "debug/policy_rejected_logits": -2.5251338481903076, | |
| "debug/policy_rejected_logps": -192.80223083496094, | |
| "debug/reference_chosen_logps": -187.92822265625, | |
| "debug/reference_rejected_logps": -192.76466369628906, | |
| "epoch": 0.06060606060606061, | |
| "grad_norm": 3.2222630610295133, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.612560987472534, | |
| "logits/rejected": -2.5251338481903076, | |
| "logps/chosen": -187.98855590820312, | |
| "logps/rejected": -192.80223083496094, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0006031226948834956, | |
| "rewards/margins": -0.00022733688820153475, | |
| "rewards/rejected": -0.0003757858066819608, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5202717781066895, | |
| "debug/policy_chosen_logps": -172.94644165039062, | |
| "debug/policy_rejected_logits": -2.446408748626709, | |
| "debug/policy_rejected_logps": -182.64926147460938, | |
| "debug/reference_chosen_logps": -172.74356079101562, | |
| "debug/reference_rejected_logps": -182.5146484375, | |
| "epoch": 0.09090909090909091, | |
| "grad_norm": 3.296231138476821, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5202717781066895, | |
| "logits/rejected": -2.446408748626709, | |
| "logps/chosen": -172.94644165039062, | |
| "logps/rejected": -182.64926147460938, | |
| "loss": 0.4995, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.002028884831815958, | |
| "rewards/margins": -0.0006827162578701973, | |
| "rewards/rejected": -0.001346168457530439, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5300447940826416, | |
| "debug/policy_chosen_logps": -180.87469482421875, | |
| "debug/policy_rejected_logits": -2.5812830924987793, | |
| "debug/policy_rejected_logps": -196.6584014892578, | |
| "debug/reference_chosen_logps": -180.65478515625, | |
| "debug/reference_rejected_logps": -196.28567504882812, | |
| "epoch": 0.12121212121212122, | |
| "grad_norm": 2.654852833868078, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5300447940826416, | |
| "logits/rejected": -2.5812830924987793, | |
| "logps/chosen": -180.87469482421875, | |
| "logps/rejected": -196.6584014892578, | |
| "loss": 0.4987, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0021991347894072533, | |
| "rewards/margins": 0.0015281867235898972, | |
| "rewards/rejected": -0.0037273215129971504, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.443774700164795, | |
| "debug/policy_chosen_logps": -189.18003845214844, | |
| "debug/policy_rejected_logits": -2.4312195777893066, | |
| "debug/policy_rejected_logps": -172.596923828125, | |
| "debug/reference_chosen_logps": -188.87039184570312, | |
| "debug/reference_rejected_logps": -172.26124572753906, | |
| "epoch": 0.15151515151515152, | |
| "grad_norm": 2.765216242619178, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.443774700164795, | |
| "logits/rejected": -2.4312195777893066, | |
| "logps/chosen": -189.18003845214844, | |
| "logps/rejected": -172.596923828125, | |
| "loss": 0.4978, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.0030963707249611616, | |
| "rewards/margins": 0.00026035308837890625, | |
| "rewards/rejected": -0.003356723580509424, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5867741107940674, | |
| "debug/policy_chosen_logps": -191.50296020507812, | |
| "debug/policy_rejected_logits": -2.5049338340759277, | |
| "debug/policy_rejected_logps": -172.64727783203125, | |
| "debug/reference_chosen_logps": -191.35916137695312, | |
| "debug/reference_rejected_logps": -171.7898406982422, | |
| "epoch": 0.18181818181818182, | |
| "grad_norm": 2.98493127681766, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5867741107940674, | |
| "logits/rejected": -2.5049338340759277, | |
| "logps/chosen": -191.50296020507812, | |
| "logps/rejected": -172.64727783203125, | |
| "loss": 0.4949, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.0014381217770278454, | |
| "rewards/margins": 0.007136192638427019, | |
| "rewards/rejected": -0.008574314415454865, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.4890458583831787, | |
| "debug/policy_chosen_logps": -188.74664306640625, | |
| "debug/policy_rejected_logits": -2.503746747970581, | |
| "debug/policy_rejected_logps": -182.01272583007812, | |
| "debug/reference_chosen_logps": -188.08114624023438, | |
| "debug/reference_rejected_logps": -181.43170166015625, | |
| "epoch": 0.21212121212121213, | |
| "grad_norm": 2.565299522305327, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.4890458583831787, | |
| "logits/rejected": -2.503746747970581, | |
| "logps/chosen": -188.74664306640625, | |
| "logps/rejected": -182.01272583007812, | |
| "loss": 0.4962, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.006655101664364338, | |
| "rewards/margins": -0.0008448789012618363, | |
| "rewards/rejected": -0.005810222588479519, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5022552013397217, | |
| "debug/policy_chosen_logps": -172.44332885742188, | |
| "debug/policy_rejected_logits": -2.38063907623291, | |
| "debug/policy_rejected_logps": -179.5426025390625, | |
| "debug/reference_chosen_logps": -171.3497772216797, | |
| "debug/reference_rejected_logps": -178.29132080078125, | |
| "epoch": 0.24242424242424243, | |
| "grad_norm": 2.683548775109707, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5022552013397217, | |
| "logits/rejected": -2.38063907623291, | |
| "logps/chosen": -172.44332885742188, | |
| "logps/rejected": -179.5426025390625, | |
| "loss": 0.4946, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.010935591533780098, | |
| "rewards/margins": 0.001577300950884819, | |
| "rewards/rejected": -0.012512893415987492, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.495532989501953, | |
| "debug/policy_chosen_logps": -188.021728515625, | |
| "debug/policy_rejected_logits": -2.334559202194214, | |
| "debug/policy_rejected_logps": -184.0438690185547, | |
| "debug/reference_chosen_logps": -187.92709350585938, | |
| "debug/reference_rejected_logps": -183.24673461914062, | |
| "epoch": 0.2727272727272727, | |
| "grad_norm": 2.872420779341629, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.495532989501953, | |
| "logits/rejected": -2.334559202194214, | |
| "logps/chosen": -188.021728515625, | |
| "logps/rejected": -184.0438690185547, | |
| "loss": 0.4937, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0009464073227718472, | |
| "rewards/margins": 0.007024994120001793, | |
| "rewards/rejected": -0.007971401326358318, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.4814565181732178, | |
| "debug/policy_chosen_logps": -198.445556640625, | |
| "debug/policy_rejected_logits": -2.496356248855591, | |
| "debug/policy_rejected_logps": -177.85418701171875, | |
| "debug/reference_chosen_logps": -198.29083251953125, | |
| "debug/reference_rejected_logps": -175.91622924804688, | |
| "epoch": 0.30303030303030304, | |
| "grad_norm": 3.0807114531930426, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.4814565181732178, | |
| "logits/rejected": -2.496356248855591, | |
| "logps/chosen": -198.445556640625, | |
| "logps/rejected": -177.85418701171875, | |
| "loss": 0.4914, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0015472793020308018, | |
| "rewards/margins": 0.01783212646842003, | |
| "rewards/rejected": -0.019379405304789543, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5934786796569824, | |
| "debug/policy_chosen_logps": -204.08248901367188, | |
| "debug/policy_rejected_logits": -2.4578843116760254, | |
| "debug/policy_rejected_logps": -186.8817138671875, | |
| "debug/reference_chosen_logps": -203.44688415527344, | |
| "debug/reference_rejected_logps": -187.01136779785156, | |
| "epoch": 0.3333333333333333, | |
| "grad_norm": 2.6552171041535537, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5934786796569824, | |
| "logits/rejected": -2.4578843116760254, | |
| "logps/chosen": -204.08248901367188, | |
| "logps/rejected": -186.8817138671875, | |
| "loss": 0.4876, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.006356086581945419, | |
| "rewards/margins": -0.0076524922624230385, | |
| "rewards/rejected": 0.001296405796892941, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5135629177093506, | |
| "debug/policy_chosen_logps": -177.83209228515625, | |
| "debug/policy_rejected_logits": -2.414395332336426, | |
| "debug/policy_rejected_logps": -174.76602172851562, | |
| "debug/reference_chosen_logps": -179.32247924804688, | |
| "debug/reference_rejected_logps": -171.77493286132812, | |
| "epoch": 0.36363636363636365, | |
| "grad_norm": 2.61493183338989, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5135629177093506, | |
| "logits/rejected": -2.414395332336426, | |
| "logps/chosen": -177.83209228515625, | |
| "logps/rejected": -174.76602172851562, | |
| "loss": 0.4916, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.014903735369443893, | |
| "rewards/margins": 0.04481462389230728, | |
| "rewards/rejected": -0.029910888522863388, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.533503532409668, | |
| "debug/policy_chosen_logps": -183.77186584472656, | |
| "debug/policy_rejected_logits": -2.3716490268707275, | |
| "debug/policy_rejected_logps": -182.7346954345703, | |
| "debug/reference_chosen_logps": -186.3679656982422, | |
| "debug/reference_rejected_logps": -182.13394165039062, | |
| "epoch": 0.3939393939393939, | |
| "grad_norm": 2.664094734451926, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.533503532409668, | |
| "logits/rejected": -2.3716490268707275, | |
| "logps/chosen": -183.77186584472656, | |
| "logps/rejected": -182.7346954345703, | |
| "loss": 0.4855, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.025960901752114296, | |
| "rewards/margins": 0.03196847438812256, | |
| "rewards/rejected": -0.006007576361298561, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.594430923461914, | |
| "debug/policy_chosen_logps": -180.72747802734375, | |
| "debug/policy_rejected_logits": -2.5284759998321533, | |
| "debug/policy_rejected_logps": -187.50599670410156, | |
| "debug/reference_chosen_logps": -182.18939208984375, | |
| "debug/reference_rejected_logps": -188.8681182861328, | |
| "epoch": 0.42424242424242425, | |
| "grad_norm": 2.5123607464425324, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.594430923461914, | |
| "logits/rejected": -2.5284759998321533, | |
| "logps/chosen": -180.72747802734375, | |
| "logps/rejected": -187.50599670410156, | |
| "loss": 0.4808, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.014619007706642151, | |
| "rewards/margins": 0.0009978096932172775, | |
| "rewards/rejected": 0.013621198013424873, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5420515537261963, | |
| "debug/policy_chosen_logps": -177.52317810058594, | |
| "debug/policy_rejected_logits": -2.487260580062866, | |
| "debug/policy_rejected_logps": -197.84640502929688, | |
| "debug/reference_chosen_logps": -178.9560089111328, | |
| "debug/reference_rejected_logps": -197.4163818359375, | |
| "epoch": 0.45454545454545453, | |
| "grad_norm": 2.680073567478649, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5420515537261963, | |
| "logits/rejected": -2.487260580062866, | |
| "logps/chosen": -177.52317810058594, | |
| "logps/rejected": -197.84640502929688, | |
| "loss": 0.4794, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.014328251592814922, | |
| "rewards/margins": 0.018628500401973724, | |
| "rewards/rejected": -0.004300250671803951, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5443243980407715, | |
| "debug/policy_chosen_logps": -175.14971923828125, | |
| "debug/policy_rejected_logits": -2.615178108215332, | |
| "debug/policy_rejected_logps": -195.92684936523438, | |
| "debug/reference_chosen_logps": -177.9427490234375, | |
| "debug/reference_rejected_logps": -192.58816528320312, | |
| "epoch": 0.48484848484848486, | |
| "grad_norm": 2.7557266784436454, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5443243980407715, | |
| "logits/rejected": -2.615178108215332, | |
| "logps/chosen": -175.14971923828125, | |
| "logps/rejected": -195.92684936523438, | |
| "loss": 0.4813, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.027930183336138725, | |
| "rewards/margins": 0.061316829174757004, | |
| "rewards/rejected": -0.03338664770126343, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.4633026123046875, | |
| "debug/policy_chosen_logps": -189.36892700195312, | |
| "debug/policy_rejected_logits": -2.456451654434204, | |
| "debug/policy_rejected_logps": -171.43634033203125, | |
| "debug/reference_chosen_logps": -192.05772399902344, | |
| "debug/reference_rejected_logps": -170.10711669921875, | |
| "epoch": 0.5151515151515151, | |
| "grad_norm": 2.7126032387088785, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.4633026123046875, | |
| "logits/rejected": -2.456451654434204, | |
| "logps/chosen": -189.36892700195312, | |
| "logps/rejected": -171.43634033203125, | |
| "loss": 0.5013, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.02688794955611229, | |
| "rewards/margins": 0.04018016904592514, | |
| "rewards/rejected": -0.013292218558490276, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.542309284210205, | |
| "debug/policy_chosen_logps": -178.5828857421875, | |
| "debug/policy_rejected_logits": -2.5415501594543457, | |
| "debug/policy_rejected_logps": -202.34744262695312, | |
| "debug/reference_chosen_logps": -180.57891845703125, | |
| "debug/reference_rejected_logps": -201.40545654296875, | |
| "epoch": 0.5454545454545454, | |
| "grad_norm": 2.604213651718002, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.542309284210205, | |
| "logits/rejected": -2.5415501594543457, | |
| "logps/chosen": -178.5828857421875, | |
| "logps/rejected": -202.34744262695312, | |
| "loss": 0.4857, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.01996036432683468, | |
| "rewards/margins": 0.0293801911175251, | |
| "rewards/rejected": -0.009419821202754974, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.51920223236084, | |
| "debug/policy_chosen_logps": -173.57725524902344, | |
| "debug/policy_rejected_logits": -2.4803247451782227, | |
| "debug/policy_rejected_logps": -176.664306640625, | |
| "debug/reference_chosen_logps": -171.49952697753906, | |
| "debug/reference_rejected_logps": -174.46640014648438, | |
| "epoch": 0.5757575757575758, | |
| "grad_norm": 3.001719578710194, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.51920223236084, | |
| "logits/rejected": -2.4803247451782227, | |
| "logps/chosen": -173.57725524902344, | |
| "logps/rejected": -176.664306640625, | |
| "loss": 0.4888, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.020777244120836258, | |
| "rewards/margins": 0.0012017078697681427, | |
| "rewards/rejected": -0.02197895012795925, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.60622501373291, | |
| "debug/policy_chosen_logps": -195.92251586914062, | |
| "debug/policy_rejected_logits": -2.4916000366210938, | |
| "debug/policy_rejected_logps": -182.05645751953125, | |
| "debug/reference_chosen_logps": -196.1642608642578, | |
| "debug/reference_rejected_logps": -176.87411499023438, | |
| "epoch": 0.6060606060606061, | |
| "grad_norm": 2.8616257419732705, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.60622501373291, | |
| "logits/rejected": -2.4916000366210938, | |
| "logps/chosen": -195.92251586914062, | |
| "logps/rejected": -182.05645751953125, | |
| "loss": 0.4718, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.002417393261566758, | |
| "rewards/margins": 0.05424093082547188, | |
| "rewards/rejected": -0.05182354152202606, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.449052572250366, | |
| "debug/policy_chosen_logps": -171.43643188476562, | |
| "debug/policy_rejected_logits": -2.4381000995635986, | |
| "debug/policy_rejected_logps": -179.66990661621094, | |
| "debug/reference_chosen_logps": -172.74703979492188, | |
| "debug/reference_rejected_logps": -178.99798583984375, | |
| "epoch": 0.6363636363636364, | |
| "grad_norm": 2.7027165426616646, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.449052572250366, | |
| "logits/rejected": -2.4381000995635986, | |
| "logps/chosen": -171.43643188476562, | |
| "logps/rejected": -179.66990661621094, | |
| "loss": 0.4773, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.013106021098792553, | |
| "rewards/margins": 0.019825361669063568, | |
| "rewards/rejected": -0.006719340104609728, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5421152114868164, | |
| "debug/policy_chosen_logps": -205.93255615234375, | |
| "debug/policy_rejected_logits": -2.630420446395874, | |
| "debug/policy_rejected_logps": -230.3187255859375, | |
| "debug/reference_chosen_logps": -208.52468872070312, | |
| "debug/reference_rejected_logps": -228.58242797851562, | |
| "epoch": 0.6666666666666666, | |
| "grad_norm": 2.7272923033258256, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5421152114868164, | |
| "logits/rejected": -2.630420446395874, | |
| "logps/chosen": -205.93255615234375, | |
| "logps/rejected": -230.3187255859375, | |
| "loss": 0.4876, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.02592136338353157, | |
| "rewards/margins": 0.0432843416929245, | |
| "rewards/rejected": -0.01736297458410263, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.6116299629211426, | |
| "debug/policy_chosen_logps": -183.0703125, | |
| "debug/policy_rejected_logits": -2.611067295074463, | |
| "debug/policy_rejected_logps": -210.6973876953125, | |
| "debug/reference_chosen_logps": -183.8590087890625, | |
| "debug/reference_rejected_logps": -211.12478637695312, | |
| "epoch": 0.696969696969697, | |
| "grad_norm": 2.5040109839505282, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.6116299629211426, | |
| "logits/rejected": -2.611067295074463, | |
| "logps/chosen": -183.0703125, | |
| "logps/rejected": -210.6973876953125, | |
| "loss": 0.4779, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.007887020707130432, | |
| "rewards/margins": 0.003613131120800972, | |
| "rewards/rejected": 0.004273890517652035, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5641233921051025, | |
| "debug/policy_chosen_logps": -160.5648193359375, | |
| "debug/policy_rejected_logits": -2.555621862411499, | |
| "debug/policy_rejected_logps": -203.09788513183594, | |
| "debug/reference_chosen_logps": -165.14849853515625, | |
| "debug/reference_rejected_logps": -203.03428649902344, | |
| "epoch": 0.7272727272727273, | |
| "grad_norm": 2.8849926512874946, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5641233921051025, | |
| "logits/rejected": -2.555621862411499, | |
| "logps/chosen": -160.5648193359375, | |
| "logps/rejected": -203.09788513183594, | |
| "loss": 0.4788, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.04583675414323807, | |
| "rewards/margins": 0.046472661197185516, | |
| "rewards/rejected": -0.000635908916592598, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.4982876777648926, | |
| "debug/policy_chosen_logps": -182.13987731933594, | |
| "debug/policy_rejected_logits": -2.5158534049987793, | |
| "debug/policy_rejected_logps": -197.07666015625, | |
| "debug/reference_chosen_logps": -180.93638610839844, | |
| "debug/reference_rejected_logps": -195.06378173828125, | |
| "epoch": 0.7575757575757576, | |
| "grad_norm": 2.573419249626687, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.4982876777648926, | |
| "logits/rejected": -2.5158534049987793, | |
| "logps/chosen": -182.13987731933594, | |
| "logps/rejected": -197.07666015625, | |
| "loss": 0.4723, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.012034913524985313, | |
| "rewards/margins": 0.00809389166533947, | |
| "rewards/rejected": -0.020128803327679634, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5644371509552, | |
| "debug/policy_chosen_logps": -190.55043029785156, | |
| "debug/policy_rejected_logits": -2.563399314880371, | |
| "debug/policy_rejected_logps": -203.17471313476562, | |
| "debug/reference_chosen_logps": -189.57899475097656, | |
| "debug/reference_rejected_logps": -200.677490234375, | |
| "epoch": 0.7878787878787878, | |
| "grad_norm": 2.6451303932135497, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5644371509552, | |
| "logits/rejected": -2.563399314880371, | |
| "logps/chosen": -190.55043029785156, | |
| "logps/rejected": -203.17471313476562, | |
| "loss": 0.4921, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.009714392013847828, | |
| "rewards/margins": 0.015257682651281357, | |
| "rewards/rejected": -0.02497207559645176, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5107343196868896, | |
| "debug/policy_chosen_logps": -157.84153747558594, | |
| "debug/policy_rejected_logits": -2.465113401412964, | |
| "debug/policy_rejected_logps": -193.6126708984375, | |
| "debug/reference_chosen_logps": -162.47134399414062, | |
| "debug/reference_rejected_logps": -187.2801971435547, | |
| "epoch": 0.8181818181818182, | |
| "grad_norm": 2.6477503407437504, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5107343196868896, | |
| "logits/rejected": -2.465113401412964, | |
| "logps/chosen": -157.84153747558594, | |
| "logps/rejected": -193.6126708984375, | |
| "loss": 0.4575, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.04629802703857422, | |
| "rewards/margins": 0.10962266474962234, | |
| "rewards/rejected": -0.06332464516162872, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.6364948749542236, | |
| "debug/policy_chosen_logps": -170.59153747558594, | |
| "debug/policy_rejected_logits": -2.5919628143310547, | |
| "debug/policy_rejected_logps": -202.32334899902344, | |
| "debug/reference_chosen_logps": -173.39129638671875, | |
| "debug/reference_rejected_logps": -198.57765197753906, | |
| "epoch": 0.8484848484848485, | |
| "grad_norm": 3.545236381819129, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.6364948749542236, | |
| "logits/rejected": -2.5919628143310547, | |
| "logps/chosen": -170.59153747558594, | |
| "logps/rejected": -202.32334899902344, | |
| "loss": 0.4874, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.02799762599170208, | |
| "rewards/margins": 0.06545457243919373, | |
| "rewards/rejected": -0.037456952035427094, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.543713331222534, | |
| "debug/policy_chosen_logps": -171.46937561035156, | |
| "debug/policy_rejected_logits": -2.5579335689544678, | |
| "debug/policy_rejected_logps": -209.11444091796875, | |
| "debug/reference_chosen_logps": -172.6361083984375, | |
| "debug/reference_rejected_logps": -201.7142333984375, | |
| "epoch": 0.8787878787878788, | |
| "grad_norm": 3.1668304273549825, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.543713331222534, | |
| "logits/rejected": -2.5579335689544678, | |
| "logps/chosen": -171.46937561035156, | |
| "logps/rejected": -209.11444091796875, | |
| "loss": 0.4725, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.011667365208268166, | |
| "rewards/margins": 0.08566955476999283, | |
| "rewards/rejected": -0.07400219142436981, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5963022708892822, | |
| "debug/policy_chosen_logps": -185.55584716796875, | |
| "debug/policy_rejected_logits": -2.591168165206909, | |
| "debug/policy_rejected_logps": -198.93161010742188, | |
| "debug/reference_chosen_logps": -189.76705932617188, | |
| "debug/reference_rejected_logps": -198.31280517578125, | |
| "epoch": 0.9090909090909091, | |
| "grad_norm": 2.9274538153529783, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5963022708892822, | |
| "logits/rejected": -2.591168165206909, | |
| "logps/chosen": -185.55584716796875, | |
| "logps/rejected": -198.93161010742188, | |
| "loss": 0.4708, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0421120822429657, | |
| "rewards/margins": 0.04830005392432213, | |
| "rewards/rejected": -0.0061879754066467285, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.540618419647217, | |
| "debug/policy_chosen_logps": -188.67941284179688, | |
| "debug/policy_rejected_logits": -2.491089105606079, | |
| "debug/policy_rejected_logps": -194.14166259765625, | |
| "debug/reference_chosen_logps": -196.98406982421875, | |
| "debug/reference_rejected_logps": -196.64881896972656, | |
| "epoch": 0.9393939393939394, | |
| "grad_norm": 2.76888166586592, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.540618419647217, | |
| "logits/rejected": -2.491089105606079, | |
| "logps/chosen": -188.67941284179688, | |
| "logps/rejected": -194.14166259765625, | |
| "loss": 0.474, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08304648846387863, | |
| "rewards/margins": 0.05797498673200607, | |
| "rewards/rejected": 0.025071505457162857, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.5790419578552246, | |
| "debug/policy_chosen_logps": -173.6497344970703, | |
| "debug/policy_rejected_logits": -2.4907901287078857, | |
| "debug/policy_rejected_logps": -215.8475799560547, | |
| "debug/reference_chosen_logps": -178.68959045410156, | |
| "debug/reference_rejected_logps": -208.4277801513672, | |
| "epoch": 0.9696969696969697, | |
| "grad_norm": 2.612964150864111, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.5790419578552246, | |
| "logits/rejected": -2.4907901287078857, | |
| "logps/chosen": -173.6497344970703, | |
| "logps/rejected": -215.8475799560547, | |
| "loss": 0.4595, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0503985770046711, | |
| "rewards/margins": 0.12459664791822433, | |
| "rewards/rejected": -0.07419807463884354, | |
| "step": 32 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": -2.553539991378784, | |
| "debug/policy_chosen_logps": -181.01400756835938, | |
| "debug/policy_rejected_logits": -2.5499818325042725, | |
| "debug/policy_rejected_logps": -200.9937286376953, | |
| "debug/reference_chosen_logps": -185.49740600585938, | |
| "debug/reference_rejected_logps": -197.2001953125, | |
| "epoch": 1.0, | |
| "grad_norm": 2.5944736669447344, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": -2.553539991378784, | |
| "logits/rejected": -2.5499818325042725, | |
| "logps/chosen": -181.01400756835938, | |
| "logps/rejected": -200.9937286376953, | |
| "loss": 0.429, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.04483398422598839, | |
| "rewards/margins": 0.08276933431625366, | |
| "rewards/rejected": -0.03793535381555557, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 33, | |
| "total_flos": 0.0, | |
| "train_loss": 0.48355864485104877, | |
| "train_runtime": 407.656, | |
| "train_samples_per_second": 5.078, | |
| "train_steps_per_second": 0.081 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 33, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |