Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a0942bef8d2562bf3d14e32ae16694c33873f9518ae5c726636253a103c96c8
+oid sha256:306d5b86e11d8f2bccd197df6210782cad275c6f5e9b618f439b7dd9f3446fa3
 size 341314196

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa400507f41a7301b7f423d1e25433d9f07068c3f05da4f2f7e6715cf2fe4994
+oid sha256:494c2d59e85fd8b0880fe6f2466dd85baf1a6957df57df23ed3e604fb4fb9672
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6676fe28230ae15b45fb334c871c6fdf1a7984a935952b9f8650896c37a8c106
+oid sha256:b1df0528620c07325b8faa7567e59b0c1e86a1f1ee6af1245a69c6c0463fe4e2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 1.1126564673157162,
+  "epoch": 2.2253129346314324,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 6.356,
       "eval_steps_per_second": 1.589,
       "step": 25
+    },
+    {
+      "epoch": 1.157162726008345,
+      "grad_norm": NaN,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 1.2016689847009736,
+      "grad_norm": NaN,
+      "learning_rate": 6.167226819279528e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 1.2461752433936022,
+      "grad_norm": NaN,
+      "learning_rate": 5.782172325201155e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 1.290681502086231,
+      "grad_norm": NaN,
+      "learning_rate": 5.392295478639225e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 1.3351877607788596,
+      "grad_norm": NaN,
+      "learning_rate": 5e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 1.3796940194714882,
+      "grad_norm": NaN,
+      "learning_rate": 4.607704521360776e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 1.4242002781641168,
+      "grad_norm": NaN,
+      "learning_rate": 4.2178276747988446e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 1.4687065368567453,
+      "grad_norm": NaN,
+      "learning_rate": 3.832773180720475e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 1.5132127955493742,
+      "grad_norm": NaN,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 1.5577190542420027,
+      "grad_norm": NaN,
+      "learning_rate": 3.086582838174551e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 1.6022253129346313,
+      "grad_norm": NaN,
+      "learning_rate": 2.7300475013022663e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 1.6467315716272601,
+      "grad_norm": NaN,
+      "learning_rate": 2.3875071764202563e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 1.6912378303198887,
+      "grad_norm": NaN,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 1.7357440890125173,
+      "grad_norm": NaN,
+      "learning_rate": 1.7527597583490822e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 1.7802503477051461,
+      "grad_norm": NaN,
+      "learning_rate": 1.4644660940672627e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 1.8247566063977747,
+      "grad_norm": NaN,
+      "learning_rate": 1.1979701719998453e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 1.8692628650904033,
+      "grad_norm": NaN,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 1.913769123783032,
+      "grad_norm": NaN,
+      "learning_rate": 7.367991782295391e-06,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 1.9582753824756607,
+      "grad_norm": NaN,
+      "learning_rate": 5.449673790581611e-06,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 2.0027816411682893,
+      "grad_norm": NaN,
+      "learning_rate": 3.8060233744356633e-06,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 2.047287899860918,
+      "grad_norm": NaN,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 2.0917941585535464,
+      "grad_norm": NaN,
+      "learning_rate": 1.3815039801161721e-06,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 2.1363004172461753,
+      "grad_norm": NaN,
+      "learning_rate": 6.15582970243117e-07,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 2.180806675938804,
+      "grad_norm": NaN,
+      "learning_rate": 1.5413331334360182e-07,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 2.2253129346314324,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 2.2253129346314324,
+      "eval_loss": NaN,
+      "eval_runtime": 23.8337,
+      "eval_samples_per_second": 6.378,
+      "eval_steps_per_second": 1.594,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.047197429066629e+17,
+  "total_flos": 6.101937426026988e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null