Training in progress, step 7000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:577ecc5f3164ed1f8764ca8bcc0923b04a67d26eb58b25b5bbfd3a6671922b0c
+oid sha256:c8b93e7897bc3f43ea6b1a664fc0daee99a1df27feebf5e9743f2a0bbfb65f30
 size 2524414853

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca12f11ddd482d1635bc0d81ed113f4ea1e018a2f62477116cfcc83538540b81
+oid sha256:b4c04132f7025e27d9cf890f119fc19b9fbfc35fa926c5345bb77dcfcf19e918
 size 1262229869

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e610a7a1ab34f2269d078954a068657d20de039358e571630da376d40a0a0adb
+oid sha256:aebbece3849c4ea65c57e14ac7218389ce4023006361ce66491299183a563542
 size 14639

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:508ae3fa47b1fb7e443f3373117b2addb3ea1903f0b754d943fc68d31b332e19
+oid sha256:9e97f75ec4a68ce2d88e985bf5af3c9d9064b2fdb2fc3093a5ded26ae274c6de
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.06791821116824107,
   "best_model_checkpoint": "./wav2vec2-xls-r-300m-ar/checkpoint-6000",
-  "epoch": 53.097345132743364,
-  "global_step": 6000,
+  "epoch": 61.94690265486726,
+  "global_step": 7000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1500,11 +1500,260 @@
       "eval_steps_per_second": 1.209,
       "eval_wer": 0.06791821116824107,
       "step": 6000
+    },
+    {
+      "epoch": 53.32,
+      "learning_rate": 0.0001255263157894737,
+      "loss": 0.0153,
+      "step": 6025
+    },
+    {
+      "epoch": 53.54,
+      "learning_rate": 0.00012473684210526315,
+      "loss": 0.013,
+      "step": 6050
+    },
+    {
+      "epoch": 53.76,
+      "learning_rate": 0.00012394736842105263,
+      "loss": 0.0093,
+      "step": 6075
+    },
+    {
+      "epoch": 53.98,
+      "learning_rate": 0.00012315789473684208,
+      "loss": 0.0125,
+      "step": 6100
+    },
+    {
+      "epoch": 54.2,
+      "learning_rate": 0.00012236842105263157,
+      "loss": 0.0122,
+      "step": 6125
+    },
+    {
+      "epoch": 54.42,
+      "learning_rate": 0.00012157894736842104,
+      "loss": 0.0131,
+      "step": 6150
+    },
+    {
+      "epoch": 54.65,
+      "learning_rate": 0.0001207894736842105,
+      "loss": 0.0129,
+      "step": 6175
+    },
+    {
+      "epoch": 54.87,
+      "learning_rate": 0.00011999999999999999,
+      "loss": 0.0119,
+      "step": 6200
+    },
+    {
+      "epoch": 55.09,
+      "learning_rate": 0.00011921052631578947,
+      "loss": 0.012,
+      "step": 6225
+    },
+    {
+      "epoch": 55.31,
+      "learning_rate": 0.00011842105263157894,
+      "loss": 0.0106,
+      "step": 6250
+    },
+    {
+      "epoch": 55.53,
+      "learning_rate": 0.00011763157894736841,
+      "loss": 0.0077,
+      "step": 6275
+    },
+    {
+      "epoch": 55.75,
+      "learning_rate": 0.00011684210526315788,
+      "loss": 0.0093,
+      "step": 6300
+    },
+    {
+      "epoch": 55.97,
+      "learning_rate": 0.00011605263157894735,
+      "loss": 0.0103,
+      "step": 6325
+    },
+    {
+      "epoch": 56.19,
+      "learning_rate": 0.00011526315789473682,
+      "loss": 0.0148,
+      "step": 6350
+    },
+    {
+      "epoch": 56.42,
+      "learning_rate": 0.00011447368421052632,
+      "loss": 0.0158,
+      "step": 6375
+    },
+    {
+      "epoch": 56.64,
+      "learning_rate": 0.00011368421052631579,
+      "loss": 0.0108,
+      "step": 6400
+    },
+    {
+      "epoch": 56.86,
+      "learning_rate": 0.00011289473684210526,
+      "loss": 0.0135,
+      "step": 6425
+    },
+    {
+      "epoch": 57.08,
+      "learning_rate": 0.00011210526315789472,
+      "loss": 0.0116,
+      "step": 6450
+    },
+    {
+      "epoch": 57.3,
+      "learning_rate": 0.0001113157894736842,
+      "loss": 0.0105,
+      "step": 6475
+    },
+    {
+      "epoch": 57.52,
+      "learning_rate": 0.00011052631578947366,
+      "loss": 0.0101,
+      "step": 6500
+    },
+    {
+      "epoch": 57.74,
+      "learning_rate": 0.00010973684210526316,
+      "loss": 0.0089,
+      "step": 6525
+    },
+    {
+      "epoch": 57.96,
+      "learning_rate": 0.00010894736842105263,
+      "loss": 0.0098,
+      "step": 6550
+    },
+    {
+      "epoch": 58.19,
+      "learning_rate": 0.0001081578947368421,
+      "loss": 0.0121,
+      "step": 6575
+    },
+    {
+      "epoch": 58.41,
+      "learning_rate": 0.00010736842105263157,
+      "loss": 0.013,
+      "step": 6600
+    },
+    {
+      "epoch": 58.63,
+      "learning_rate": 0.00010657894736842104,
+      "loss": 0.012,
+      "step": 6625
+    },
+    {
+      "epoch": 58.85,
+      "learning_rate": 0.00010578947368421051,
+      "loss": 0.0089,
+      "step": 6650
+    },
+    {
+      "epoch": 59.07,
+      "learning_rate": 0.00010499999999999999,
+      "loss": 0.0111,
+      "step": 6675
+    },
+    {
+      "epoch": 59.29,
+      "learning_rate": 0.00010421052631578947,
+      "loss": 0.0085,
+      "step": 6700
+    },
+    {
+      "epoch": 59.51,
+      "learning_rate": 0.00010342105263157894,
+      "loss": 0.0094,
+      "step": 6725
+    },
+    {
+      "epoch": 59.73,
+      "learning_rate": 0.00010263157894736841,
+      "loss": 0.0098,
+      "step": 6750
+    },
+    {
+      "epoch": 59.96,
+      "learning_rate": 0.00010184210526315788,
+      "loss": 0.0109,
+      "step": 6775
+    },
+    {
+      "epoch": 60.18,
+      "learning_rate": 0.00010105263157894735,
+      "loss": 0.0129,
+      "step": 6800
+    },
+    {
+      "epoch": 60.4,
+      "learning_rate": 0.00010026315789473683,
+      "loss": 0.01,
+      "step": 6825
+    },
+    {
+      "epoch": 60.62,
+      "learning_rate": 9.94736842105263e-05,
+      "loss": 0.0089,
+      "step": 6850
+    },
+    {
+      "epoch": 60.84,
+      "learning_rate": 9.868421052631579e-05,
+      "loss": 0.0122,
+      "step": 6875
+    },
+    {
+      "epoch": 61.06,
+      "learning_rate": 9.789473684210526e-05,
+      "loss": 0.0167,
+      "step": 6900
+    },
+    {
+      "epoch": 61.28,
+      "learning_rate": 9.710526315789473e-05,
+      "loss": 0.0105,
+      "step": 6925
+    },
+    {
+      "epoch": 61.5,
+      "learning_rate": 9.63157894736842e-05,
+      "loss": 0.0089,
+      "step": 6950
+    },
+    {
+      "epoch": 61.73,
+      "learning_rate": 9.552631578947368e-05,
+      "loss": 0.0102,
+      "step": 6975
+    },
+    {
+      "epoch": 61.95,
+      "learning_rate": 9.473684210526315e-05,
+      "loss": 0.0149,
+      "step": 7000
+    },
+    {
+      "epoch": 61.95,
+      "eval_loss": 0.2759546935558319,
+      "eval_runtime": 10.7835,
+      "eval_samples_per_second": 9.273,
+      "eval_steps_per_second": 1.206,
+      "eval_wer": 0.07174458926222647,
+      "step": 7000
     }
   ],
   "max_steps": 10000,
   "num_train_epochs": 89,
-  "total_flos": 2.3640957156336247e+19,
+  "total_flos": 2.756451071078706e+19,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca12f11ddd482d1635bc0d81ed113f4ea1e018a2f62477116cfcc83538540b81
+oid sha256:b4c04132f7025e27d9cf890f119fc19b9fbfc35fa926c5345bb77dcfcf19e918
 size 1262229869

runs/Apr09_14-45-21_gpu07.cyverse.org/events.out.tfevents.1681076808.gpu07.cyverse.org.48142.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fa0c2b70002f87a44d106c5be0907e96fc3567270c9387d4e18c661e3204a4b
-size 44970
+oid sha256:17afcb646f4bffba0651ee2b2d287faa8b2e8e9a98dac1f49fdb1210b9df2b8a
+size 51568