Training in progress, step 8000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8b93e7897bc3f43ea6b1a664fc0daee99a1df27feebf5e9743f2a0bbfb65f30
+oid sha256:01ff7248d086cb5c4cedadeb579adbb675a1c64e4fb09e5ae750359a44febb76
 size 2524414853

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4c04132f7025e27d9cf890f119fc19b9fbfc35fa926c5345bb77dcfcf19e918
+oid sha256:4607f812b37e97d000dcc622b3e637fb3031eca1d8fd3c7555cdaadfec40180b
 size 1262229869

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aebbece3849c4ea65c57e14ac7218389ce4023006361ce66491299183a563542
+oid sha256:977a8fef6282492c80e91318169668264512de99e37019c60391b3f14bdd2233
 size 14639

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e97f75ec4a68ce2d88e985bf5af3c9d9064b2fdb2fc3093a5ded26ae274c6de
+oid sha256:e3c549db73c256944038c972af144f55d1e0fbea4d86305beed62e57f9cae640
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.06791821116824107,
   "best_model_checkpoint": "./wav2vec2-xls-r-300m-ar/checkpoint-6000",
-  "epoch": 61.94690265486726,
-  "global_step": 7000,
+  "epoch": 70.79646017699115,
+  "global_step": 8000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1749,11 +1749,260 @@
       "eval_steps_per_second": 1.206,
       "eval_wer": 0.07174458926222647,
       "step": 7000
+    },
+    {
+      "epoch": 62.17,
+      "learning_rate": 9.394736842105262e-05,
+      "loss": 0.0116,
+      "step": 7025
+    },
+    {
+      "epoch": 62.39,
+      "learning_rate": 9.31578947368421e-05,
+      "loss": 0.0102,
+      "step": 7050
+    },
+    {
+      "epoch": 62.61,
+      "learning_rate": 9.236842105263157e-05,
+      "loss": 0.0118,
+      "step": 7075
+    },
+    {
+      "epoch": 62.83,
+      "learning_rate": 9.157894736842104e-05,
+      "loss": 0.0093,
+      "step": 7100
+    },
+    {
+      "epoch": 63.05,
+      "learning_rate": 9.078947368421052e-05,
+      "loss": 0.0074,
+      "step": 7125
+    },
+    {
+      "epoch": 63.27,
+      "learning_rate": 8.999999999999999e-05,
+      "loss": 0.0079,
+      "step": 7150
+    },
+    {
+      "epoch": 63.5,
+      "learning_rate": 8.921052631578946e-05,
+      "loss": 0.0075,
+      "step": 7175
+    },
+    {
+      "epoch": 63.72,
+      "learning_rate": 8.842105263157893e-05,
+      "loss": 0.0099,
+      "step": 7200
+    },
+    {
+      "epoch": 63.94,
+      "learning_rate": 8.763157894736841e-05,
+      "loss": 0.0098,
+      "step": 7225
+    },
+    {
+      "epoch": 64.16,
+      "learning_rate": 8.68421052631579e-05,
+      "loss": 0.0109,
+      "step": 7250
+    },
+    {
+      "epoch": 64.38,
+      "learning_rate": 8.605263157894737e-05,
+      "loss": 0.0103,
+      "step": 7275
+    },
+    {
+      "epoch": 64.6,
+      "learning_rate": 8.526315789473684e-05,
+      "loss": 0.0074,
+      "step": 7300
+    },
+    {
+      "epoch": 64.82,
+      "learning_rate": 8.44736842105263e-05,
+      "loss": 0.0101,
+      "step": 7325
+    },
+    {
+      "epoch": 65.04,
+      "learning_rate": 8.368421052631578e-05,
+      "loss": 0.009,
+      "step": 7350
+    },
+    {
+      "epoch": 65.27,
+      "learning_rate": 8.289473684210526e-05,
+      "loss": 0.0077,
+      "step": 7375
+    },
+    {
+      "epoch": 65.49,
+      "learning_rate": 8.210526315789474e-05,
+      "loss": 0.009,
+      "step": 7400
+    },
+    {
+      "epoch": 65.71,
+      "learning_rate": 8.131578947368421e-05,
+      "loss": 0.0098,
+      "step": 7425
+    },
+    {
+      "epoch": 65.93,
+      "learning_rate": 8.052631578947368e-05,
+      "loss": 0.0096,
+      "step": 7450
+    },
+    {
+      "epoch": 66.15,
+      "learning_rate": 7.973684210526315e-05,
+      "loss": 0.0136,
+      "step": 7475
+    },
+    {
+      "epoch": 66.37,
+      "learning_rate": 7.894736842105262e-05,
+      "loss": 0.0082,
+      "step": 7500
+    },
+    {
+      "epoch": 66.59,
+      "learning_rate": 7.815789473684209e-05,
+      "loss": 0.0089,
+      "step": 7525
+    },
+    {
+      "epoch": 66.81,
+      "learning_rate": 7.736842105263159e-05,
+      "loss": 0.0089,
+      "step": 7550
+    },
+    {
+      "epoch": 67.04,
+      "learning_rate": 7.657894736842105e-05,
+      "loss": 0.0084,
+      "step": 7575
+    },
+    {
+      "epoch": 67.26,
+      "learning_rate": 7.578947368421052e-05,
+      "loss": 0.0073,
+      "step": 7600
+    },
+    {
+      "epoch": 67.48,
+      "learning_rate": 7.5e-05,
+      "loss": 0.008,
+      "step": 7625
+    },
+    {
+      "epoch": 67.7,
+      "learning_rate": 7.421052631578946e-05,
+      "loss": 0.0083,
+      "step": 7650
+    },
+    {
+      "epoch": 67.92,
+      "learning_rate": 7.342105263157895e-05,
+      "loss": 0.0068,
+      "step": 7675
+    },
+    {
+      "epoch": 68.14,
+      "learning_rate": 7.263157894736842e-05,
+      "loss": 0.007,
+      "step": 7700
+    },
+    {
+      "epoch": 68.36,
+      "learning_rate": 7.184210526315788e-05,
+      "loss": 0.0086,
+      "step": 7725
+    },
+    {
+      "epoch": 68.58,
+      "learning_rate": 7.105263157894735e-05,
+      "loss": 0.0077,
+      "step": 7750
+    },
+    {
+      "epoch": 68.81,
+      "learning_rate": 7.026315789473684e-05,
+      "loss": 0.0076,
+      "step": 7775
+    },
+    {
+      "epoch": 69.03,
+      "learning_rate": 6.947368421052631e-05,
+      "loss": 0.006,
+      "step": 7800
+    },
+    {
+      "epoch": 69.25,
+      "learning_rate": 6.868421052631578e-05,
+      "loss": 0.006,
+      "step": 7825
+    },
+    {
+      "epoch": 69.47,
+      "learning_rate": 6.789473684210526e-05,
+      "loss": 0.0077,
+      "step": 7850
+    },
+    {
+      "epoch": 69.69,
+      "learning_rate": 6.710526315789473e-05,
+      "loss": 0.009,
+      "step": 7875
+    },
+    {
+      "epoch": 69.91,
+      "learning_rate": 6.63157894736842e-05,
+      "loss": 0.0081,
+      "step": 7900
+    },
+    {
+      "epoch": 70.13,
+      "learning_rate": 6.552631578947368e-05,
+      "loss": 0.0085,
+      "step": 7925
+    },
+    {
+      "epoch": 70.35,
+      "learning_rate": 6.473684210526315e-05,
+      "loss": 0.007,
+      "step": 7950
+    },
+    {
+      "epoch": 70.58,
+      "learning_rate": 6.394736842105262e-05,
+      "loss": 0.0088,
+      "step": 7975
+    },
+    {
+      "epoch": 70.8,
+      "learning_rate": 6.315789473684209e-05,
+      "loss": 0.0074,
+      "step": 8000
+    },
+    {
+      "epoch": 70.8,
+      "eval_loss": 0.30043184757232666,
+      "eval_runtime": 10.795,
+      "eval_samples_per_second": 9.264,
+      "eval_steps_per_second": 1.204,
+      "eval_wer": 0.0680377854836781,
+      "step": 8000
     }
   ],
   "max_steps": 10000,
   "num_train_epochs": 89,
-  "total_flos": 2.756451071078706e+19,
+  "total_flos": 3.1517992803917713e+19,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4c04132f7025e27d9cf890f119fc19b9fbfc35fa926c5345bb77dcfcf19e918
+oid sha256:4607f812b37e97d000dcc622b3e637fb3031eca1d8fd3c7555cdaadfec40180b
 size 1262229869

runs/Apr09_14-45-21_gpu07.cyverse.org/events.out.tfevents.1681076808.gpu07.cyverse.org.48142.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17afcb646f4bffba0651ee2b2d287faa8b2e8e9a98dac1f49fdb1210b9df2b8a
-size 51568
+oid sha256:14933acd55b5be1858479eeffcc145290dda1fac71c2c315cec1bd00af72f3df
+size 58166