trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 48,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "learning_rate": 1e-07,
      "logits/chosen": -2.8253421783447266,
      "logits/rejected": -2.860417366027832,
      "logps/chosen": -254.3809356689453,
      "logps/pi_response": -73.23240661621094,
      "logps/ref_response": -73.23240661621094,
      "logps/rejected": -181.62828063964844,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.83504027183137e-07,
      "logits/chosen": -2.7532455921173096,
      "logits/rejected": -2.7328317165374756,
      "logps/chosen": -223.27049255371094,
      "logps/pi_response": -71.2132339477539,
      "logps/ref_response": -70.33279418945312,
      "logps/rejected": -168.04412841796875,
      "loss": 0.688,
      "rewards/accuracies": 0.5520833134651184,
      "rewards/chosen": 0.01192192081362009,
      "rewards/margins": 0.012628484517335892,
      "rewards/rejected": -0.0007065660320222378,
      "step": 10
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.643105808261596e-07,
      "logits/chosen": -2.7211954593658447,
      "logits/rejected": -2.6908421516418457,
      "logps/chosen": -256.15582275390625,
      "logps/pi_response": -83.29663848876953,
      "logps/ref_response": -70.55802917480469,
      "logps/rejected": -166.14871215820312,
      "loss": 0.655,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.02286558598279953,
      "rewards/margins": 0.08656834065914154,
      "rewards/rejected": -0.06370275467634201,
      "step": 20
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8676665440207977e-07,
      "logits/chosen": -2.638202428817749,
      "logits/rejected": -2.6029179096221924,
      "logps/chosen": -263.89013671875,
      "logps/pi_response": -118.5997543334961,
      "logps/ref_response": -77.43115234375,
      "logps/rejected": -209.71957397460938,
      "loss": 0.6233,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09839216619729996,
      "rewards/margins": 0.22308287024497986,
      "rewards/rejected": -0.321475088596344,
      "step": 30
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1500545527530544e-08,
      "logits/chosen": -2.6645946502685547,
      "logits/rejected": -2.6340341567993164,
      "logps/chosen": -276.5579833984375,
      "logps/pi_response": -136.46200561523438,
      "logps/ref_response": -80.81505584716797,
      "logps/rejected": -224.3572235107422,
      "loss": 0.6049,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.1703888475894928,
      "rewards/margins": 0.2641330361366272,
      "rewards/rejected": -0.4345219135284424,
      "step": 40
    },
    {
      "epoch": 1.0,
      "step": 48,
      "total_flos": 0.0,
      "train_loss": 0.6304036478201548,
      "train_runtime": 2878.7206,
      "train_samples_per_second": 4.247,
      "train_steps_per_second": 0.017
    }
  ],
  "logging_steps": 10,
  "max_steps": 48,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}