trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 48,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "learning_rate": 1e-07,
      "logits/chosen": -2.8253421783447266,
      "logits/rejected": -2.860417366027832,
      "logps/chosen": -254.3809356689453,
      "logps/pi_response": -73.23240661621094,
      "logps/ref_response": -73.23240661621094,
      "logps/rejected": -181.62828063964844,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.83504027183137e-07,
      "logits/chosen": -2.7526257038116455,
      "logits/rejected": -2.7321386337280273,
      "logps/chosen": -223.6942596435547,
      "logps/pi_response": -71.71171569824219,
      "logps/ref_response": -70.33279418945312,
      "logps/rejected": -168.48367309570312,
      "loss": 0.688,
      "rewards/accuracies": 0.6006944179534912,
      "rewards/chosen": 0.0076843444257974625,
      "rewards/margins": 0.012786192819476128,
      "rewards/rejected": -0.0051018488593399525,
      "step": 10
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.643105808261596e-07,
      "logits/chosen": -2.7141361236572266,
      "logits/rejected": -2.6840004920959473,
      "logps/chosen": -258.06878662109375,
      "logps/pi_response": -86.14505767822266,
      "logps/ref_response": -70.55802917480469,
      "logps/rejected": -168.45187377929688,
      "loss": 0.6523,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": 0.003736161859706044,
      "rewards/margins": 0.09047064930200577,
      "rewards/rejected": -0.08673448860645294,
      "step": 20
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8676665440207977e-07,
      "logits/chosen": -2.6236815452575684,
      "logits/rejected": -2.5884528160095215,
      "logps/chosen": -266.60638427734375,
      "logps/pi_response": -122.98347473144531,
      "logps/ref_response": -77.43115234375,
      "logps/rejected": -213.4358367919922,
      "loss": 0.6176,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12555423378944397,
      "rewards/margins": 0.23308345675468445,
      "rewards/rejected": -0.3586376905441284,
      "step": 30
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1500545527530544e-08,
      "logits/chosen": -2.6426758766174316,
      "logits/rejected": -2.6125741004943848,
      "logps/chosen": -279.7016906738281,
      "logps/pi_response": -142.2088165283203,
      "logps/ref_response": -80.81505584716797,
      "logps/rejected": -229.06588745117188,
      "loss": 0.5967,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.20182597637176514,
      "rewards/margins": 0.27978265285491943,
      "rewards/rejected": -0.4816085696220398,
      "step": 40
    },
    {
      "epoch": 1.0,
      "step": 48,
      "total_flos": 0.0,
      "train_loss": 0.6252773602803549,
      "train_runtime": 2869.3319,
      "train_samples_per_second": 4.261,
      "train_steps_per_second": 0.017
    }
  ],
  "logging_steps": 10,
  "max_steps": 48,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}