Training in progress, step 6000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d04ac14b396acc439a80242e3de2dde5c2c3cfe230aff038ac79cfc71137162a
+oid sha256:577ecc5f3164ed1f8764ca8bcc0923b04a67d26eb58b25b5bbfd3a6671922b0c
 size 2524414853

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72797f1440dbbfd35bf83a8efb707c911377db12eea28703ebe830d181f4ed7f
+oid sha256:ca12f11ddd482d1635bc0d81ed113f4ea1e018a2f62477116cfcc83538540b81
 size 1262229869

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38c34f160b4325fa8e05323d539226829df54c92e017ae64958c023e2dbecb9e
-size 14575
+oid sha256:e610a7a1ab34f2269d078954a068657d20de039358e571630da376d40a0a0adb
+size 14639

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e00e24b9ec2f00c6f283cf8c9dd0d87ed62551d6e0d1598b4aa9ff96f37c557
+oid sha256:508ae3fa47b1fb7e443f3373117b2addb3ea1903f0b754d943fc68d31b332e19
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.0693531029534856,
-  "best_model_checkpoint": "./wav2vec2-xls-r-300m-ar/checkpoint-3000",
-  "epoch": 44.24778761061947,
-  "global_step": 5000,
+  "best_metric": 0.06791821116824107,
+  "best_model_checkpoint": "./wav2vec2-xls-r-300m-ar/checkpoint-6000",
+  "epoch": 53.097345132743364,
+  "global_step": 6000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1251,11 +1251,260 @@
       "eval_steps_per_second": 1.211,
       "eval_wer": 0.07437522420184145,
       "step": 5000
+    },
+    {
+      "epoch": 44.47,
+      "learning_rate": 0.00015710526315789472,
+      "loss": 0.0163,
+      "step": 5025
+    },
+    {
+      "epoch": 44.69,
+      "learning_rate": 0.00015631578947368418,
+      "loss": 0.0196,
+      "step": 5050
+    },
+    {
+      "epoch": 44.91,
+      "learning_rate": 0.00015552631578947366,
+      "loss": 0.0161,
+      "step": 5075
+    },
+    {
+      "epoch": 45.13,
+      "learning_rate": 0.00015473684210526317,
+      "loss": 0.0158,
+      "step": 5100
+    },
+    {
+      "epoch": 45.35,
+      "learning_rate": 0.00015394736842105263,
+      "loss": 0.0159,
+      "step": 5125
+    },
+    {
+      "epoch": 45.58,
+      "learning_rate": 0.0001531578947368421,
+      "loss": 0.0163,
+      "step": 5150
+    },
+    {
+      "epoch": 45.8,
+      "learning_rate": 0.00015236842105263156,
+      "loss": 0.0137,
+      "step": 5175
+    },
+    {
+      "epoch": 46.02,
+      "learning_rate": 0.00015157894736842105,
+      "loss": 0.0131,
+      "step": 5200
+    },
+    {
+      "epoch": 46.24,
+      "learning_rate": 0.0001507894736842105,
+      "loss": 0.0125,
+      "step": 5225
+    },
+    {
+      "epoch": 46.46,
+      "learning_rate": 0.00015,
+      "loss": 0.0138,
+      "step": 5250
+    },
+    {
+      "epoch": 46.68,
+      "learning_rate": 0.00014921052631578947,
+      "loss": 0.0179,
+      "step": 5275
+    },
+    {
+      "epoch": 46.9,
+      "learning_rate": 0.00014842105263157893,
+      "loss": 0.0169,
+      "step": 5300
+    },
+    {
+      "epoch": 47.12,
+      "learning_rate": 0.0001476315789473684,
+      "loss": 0.0152,
+      "step": 5325
+    },
+    {
+      "epoch": 47.35,
+      "learning_rate": 0.0001468421052631579,
+      "loss": 0.0146,
+      "step": 5350
+    },
+    {
+      "epoch": 47.57,
+      "learning_rate": 0.00014605263157894735,
+      "loss": 0.0143,
+      "step": 5375
+    },
+    {
+      "epoch": 47.79,
+      "learning_rate": 0.00014526315789473683,
+      "loss": 0.0116,
+      "step": 5400
+    },
+    {
+      "epoch": 48.01,
+      "learning_rate": 0.00014447368421052631,
+      "loss": 0.0162,
+      "step": 5425
+    },
+    {
+      "epoch": 48.23,
+      "learning_rate": 0.00014368421052631577,
+      "loss": 0.0115,
+      "step": 5450
+    },
+    {
+      "epoch": 48.45,
+      "learning_rate": 0.00014289473684210525,
+      "loss": 0.0156,
+      "step": 5475
+    },
+    {
+      "epoch": 48.67,
+      "learning_rate": 0.0001421052631578947,
+      "loss": 0.0144,
+      "step": 5500
+    },
+    {
+      "epoch": 48.89,
+      "learning_rate": 0.0001413157894736842,
+      "loss": 0.0155,
+      "step": 5525
+    },
+    {
+      "epoch": 49.12,
+      "learning_rate": 0.00014052631578947367,
+      "loss": 0.0146,
+      "step": 5550
+    },
+    {
+      "epoch": 49.34,
+      "learning_rate": 0.00013973684210526316,
+      "loss": 0.0164,
+      "step": 5575
+    },
+    {
+      "epoch": 49.56,
+      "learning_rate": 0.00013894736842105261,
+      "loss": 0.0128,
+      "step": 5600
+    },
+    {
+      "epoch": 49.78,
+      "learning_rate": 0.0001381578947368421,
+      "loss": 0.0133,
+      "step": 5625
+    },
+    {
+      "epoch": 50.0,
+      "learning_rate": 0.00013736842105263155,
+      "loss": 0.0126,
+      "step": 5650
+    },
+    {
+      "epoch": 50.22,
+      "learning_rate": 0.00013657894736842104,
+      "loss": 0.011,
+      "step": 5675
+    },
+    {
+      "epoch": 50.44,
+      "learning_rate": 0.00013578947368421052,
+      "loss": 0.0154,
+      "step": 5700
+    },
+    {
+      "epoch": 50.66,
+      "learning_rate": 0.000135,
+      "loss": 0.0154,
+      "step": 5725
+    },
+    {
+      "epoch": 50.88,
+      "learning_rate": 0.00013421052631578946,
+      "loss": 0.0131,
+      "step": 5750
+    },
+    {
+      "epoch": 51.11,
+      "learning_rate": 0.00013342105263157894,
+      "loss": 0.0161,
+      "step": 5775
+    },
+    {
+      "epoch": 51.33,
+      "learning_rate": 0.0001326315789473684,
+      "loss": 0.0135,
+      "step": 5800
+    },
+    {
+      "epoch": 51.55,
+      "learning_rate": 0.00013184210526315788,
+      "loss": 0.0125,
+      "step": 5825
+    },
+    {
+      "epoch": 51.77,
+      "learning_rate": 0.00013105263157894736,
+      "loss": 0.0114,
+      "step": 5850
+    },
+    {
+      "epoch": 51.99,
+      "learning_rate": 0.00013026315789473685,
+      "loss": 0.0106,
+      "step": 5875
+    },
+    {
+      "epoch": 52.21,
+      "learning_rate": 0.0001294736842105263,
+      "loss": 0.0119,
+      "step": 5900
+    },
+    {
+      "epoch": 52.43,
+      "learning_rate": 0.00012868421052631578,
+      "loss": 0.0123,
+      "step": 5925
+    },
+    {
+      "epoch": 52.65,
+      "learning_rate": 0.00012789473684210524,
+      "loss": 0.0111,
+      "step": 5950
+    },
+    {
+      "epoch": 52.88,
+      "learning_rate": 0.00012710526315789472,
+      "loss": 0.0139,
+      "step": 5975
+    },
+    {
+      "epoch": 53.1,
+      "learning_rate": 0.00012631578947368418,
+      "loss": 0.0155,
+      "step": 6000
+    },
+    {
+      "epoch": 53.1,
+      "eval_loss": 0.2688598036766052,
+      "eval_runtime": 10.7568,
+      "eval_samples_per_second": 9.296,
+      "eval_steps_per_second": 1.209,
+      "eval_wer": 0.06791821116824107,
+      "step": 6000
     }
   ],
   "max_steps": 10000,
   "num_train_epochs": 89,
-  "total_flos": 1.96951110785928e+19,
+  "total_flos": 2.3640957156336247e+19,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72797f1440dbbfd35bf83a8efb707c911377db12eea28703ebe830d181f4ed7f
+oid sha256:ca12f11ddd482d1635bc0d81ed113f4ea1e018a2f62477116cfcc83538540b81
 size 1262229869

runs/Apr09_14-45-21_gpu07.cyverse.org/events.out.tfevents.1681076808.gpu07.cyverse.org.48142.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d0c176c61ff82428c1cfed0632db9bfe35c7927ea235840f13920ec7caf7126
-size 38372
+oid sha256:5fa0c2b70002f87a44d106c5be0907e96fc3567270c9387d4e18c661e3204a4b
+size 44970