Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e7b51f98c3db47f0101590d208568101ab9c74651d12b539f775434987e658f
+oid sha256:7399dd09465f4ddaaa0bf29f5a28a46b4f2a17e499395cf15fa5491589188868
 size 1342555602

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac2998e19417e38a9d71662701f223fe59a991f3de9e30a74a292fe94292ffcf
+oid sha256:c9ed913daeb3d28f759d4ef1e25121a5313dd98da92bb7336997e5cf8a20853a
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c931ece4da598a541d357f6c98f67481603252e193960022d37ddd49c584b1f
+oid sha256:b1df0528620c07325b8faa7567e59b0c1e86a1f1ee6af1245a69c6c0463fe4e2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 0.1278772378516624,
+  "epoch": 0.2557544757033248,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 13.619,
       "eval_steps_per_second": 1.733,
       "step": 25
+    },
+    {
+      "epoch": 0.1329923273657289,
+      "grad_norm": NaN,
+      "learning_rate": 9.045084971874738e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 0.13810741687979539,
+      "grad_norm": NaN,
+      "learning_rate": 8.715724127386972e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 0.1432225063938619,
+      "grad_norm": NaN,
+      "learning_rate": 8.345653031794292e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 0.1483375959079284,
+      "grad_norm": NaN,
+      "learning_rate": 7.938926261462366e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 0.1534526854219949,
+      "grad_norm": NaN,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.1585677749360614,
+      "grad_norm": NaN,
+      "learning_rate": 7.033683215379002e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 0.1636828644501279,
+      "grad_norm": NaN,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 0.16879795396419436,
+      "grad_norm": NaN,
+      "learning_rate": 6.0395584540887963e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 0.17391304347826086,
+      "grad_norm": NaN,
+      "learning_rate": 5.522642316338268e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 0.17902813299232737,
+      "grad_norm": NaN,
+      "learning_rate": 5e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 0.18414322250639387,
+      "grad_norm": NaN,
+      "learning_rate": 4.477357683661734e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 0.18925831202046037,
+      "grad_norm": NaN,
+      "learning_rate": 3.960441545911204e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 0.19437340153452684,
+      "grad_norm": NaN,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 0.19948849104859334,
+      "grad_norm": NaN,
+      "learning_rate": 2.9663167846209998e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 0.20460358056265984,
+      "grad_norm": NaN,
+      "learning_rate": 2.500000000000001e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.20971867007672634,
+      "grad_norm": NaN,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 0.21483375959079284,
+      "grad_norm": NaN,
+      "learning_rate": 1.6543469682057106e-05,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 0.21994884910485935,
+      "grad_norm": NaN,
+      "learning_rate": 1.2842758726130283e-05,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 0.22506393861892582,
+      "grad_norm": NaN,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 0.23017902813299232,
+      "grad_norm": NaN,
+      "learning_rate": 6.698729810778065e-06,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 0.23529411764705882,
+      "grad_norm": NaN,
+      "learning_rate": 4.322727117869951e-06,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 0.24040920716112532,
+      "grad_norm": NaN,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 0.24552429667519182,
+      "grad_norm": NaN,
+      "learning_rate": 1.0926199633097157e-06,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 0.2506393861892583,
+      "grad_norm": NaN,
+      "learning_rate": 2.7390523158633554e-07,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 0.2557544757033248,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.2557544757033248,
+      "eval_loss": NaN,
+      "eval_runtime": 24.2345,
+      "eval_samples_per_second": 13.617,
+      "eval_steps_per_second": 1.733,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.77128394686464e+16,
+  "total_flos": 7.54256789372928e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null