Training in progress, step 9000

.gitattributes

@@ -38,3 +38,4 @@ community-events/huggan/assets/lightweight_gan_wandb.png filter=lfs diff=lfs mer
 community-events/huggan/assets/pix2pix_maps.png filter=lfs diff=lfs merge=lfs -text
 community-events/huggan/assets/wandb.png filter=lfs diff=lfs merge=lfs -text
 wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb filter=lfs diff=lfs merge=lfs -text

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88e1983e53d9772aab424c633032c6617c2bf689500e14db84ddc851ae5c8628
+oid sha256:5943e8c30fa544ff09f628b48252853f4960d70627894b945d089a224bd8bfe1
 size 3219908024

wandb/run-20231119_105908-nwx0xdmm/files/output.log

@@ -1155,3 +1155,1072 @@ Reading metadata...: 1it [00:00,  5.99it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 13%|████████████▌                                                                                | 8079/60000 [5:01:12<137:59:55,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 13%|████████████▌                                                                                | 8099/60000 [5:04:44<141:41:08,  9.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▌                                                                                | 8119/60000 [5:08:07<145:13:16, 10.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▌                                                                                | 8139/60000 [5:11:51<187:32:02, 13.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▋                                                                                | 8159/60000 [5:15:10<140:39:12,  9.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▋                                                                                | 8179/60000 [5:18:28<141:24:04,  9.82s/it]
+
+
+
+
+
+
+
+ 14%|████████████▋                                                                                | 8186/60000 [5:19:37<141:50:31,  9.86s/it]
+
+ 14%|████████████▋                                                                                | 8187/60000 [5:19:43<127:16:19,  8.84s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▋                                                                                | 8199/60000 [5:21:35<136:36:17,  9.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▋                                                                                | 8219/60000 [5:24:47<137:18:55,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▊                                                                                | 8239/60000 [5:28:14<166:10:17, 11.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▊                                                                                | 8259/60000 [5:31:30<140:56:08,  9.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▊                                                                                | 8279/60000 [5:34:46<136:44:33,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▊                                                                                | 8299/60000 [5:38:12<163:33:15, 11.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▉                                                                                | 8320/60000 [5:41:35<138:53:27,  9.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▉                                                                                | 8339/60000 [5:44:37<136:19:21,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▉                                                                                | 8359/60000 [5:47:54<137:59:34,  9.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|████████████▉                                                                                | 8379/60000 [5:51:07<139:22:49,  9.72s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████                                                                                | 8400/60000 [5:54:27<137:15:03,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████                                                                                | 8419/60000 [5:57:31<135:38:06,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████                                                                                | 8439/60000 [6:00:49<154:22:19, 10.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████                                                                                | 8459/60000 [6:04:01<136:56:17,  9.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▏                                                                               | 8479/60000 [6:07:14<137:29:35,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▏                                                                               | 8499/60000 [6:10:28<140:54:19,  9.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▏                                                                               | 8520/60000 [6:13:51<140:34:02,  9.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▏                                                                               | 8540/60000 [6:17:21<136:48:57,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▎                                                                               | 8559/60000 [6:20:24<137:45:11,  9.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▎                                                                               | 8580/60000 [6:23:45<138:07:42,  9.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▎                                                                               | 8594/60000 [6:25:58<123:58:31,  8.68s/it]
+
+
+
+
+
+ 14%|█████████████▎                                                                               | 8599/60000 [6:26:47<136:29:25,  9.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▎                                                                               | 8619/60000 [6:30:05<144:26:38, 10.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▍                                                                               | 8639/60000 [6:33:27<137:50:45,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▍                                                                               | 8660/60000 [6:36:48<134:50:43,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▍                                                                               | 8679/60000 [6:39:50<136:05:44,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 14%|█████████████▍                                                                               | 8699/60000 [6:43:04<140:09:38,  9.84s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▌                                                                               | 8719/60000 [6:46:22<142:20:45,  9.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▌                                                                               | 8739/60000 [6:49:35<139:38:15,  9.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▌                                                                               | 8759/60000 [6:52:50<141:23:58,  9.93s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▌                                                                               | 8779/60000 [6:56:03<134:53:33,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▋                                                                               | 8799/60000 [6:59:46<149:59:39, 10.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▋                                                                               | 8819/60000 [7:03:00<137:06:16,  9.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▋                                                                               | 8839/60000 [7:06:14<137:54:27,  9.70s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▋                                                                               | 8859/60000 [7:09:29<138:16:07,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▊                                                                               | 8879/60000 [7:12:49<158:57:19, 11.19s/it]
+Reading metadata...: 1650it [00:00, 9158.08it/s]                                                  | 8880/60000 [7:12:58<151:08:54, 10.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▊                                                                               | 8900/60000 [7:16:50<139:26:23,  9.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▊                                                                               | 8919/60000 [7:19:57<142:39:26, 10.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▊                                                                               | 8939/60000 [7:23:11<139:14:39,  9.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▉                                                                               | 8959/60000 [7:26:26<136:56:34,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|████████��████▉                                                                               | 8979/60000 [7:29:55<140:01:11,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▉                                                                               | 8999/60000 [7:33:22<139:13:16,  9.83s/it]
+ 15%|█████████████▉                                                                               | 9000/60000 [7:33:32<140:19:23,  9.91s/it][INFO|trainer.py:3173] 2023-11-19 18:32:46,803 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-19 18:32:46,803 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-19 18:32:46,803 >>   Batch size = 4
+
+
+[INFO|trainer_utils.py:759] 2023-11-19 18:32:49,705 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+ 15%|█████████████▉                                                                               | 9000/60000 [7:43:57<140:19:23,  9.91s/it]
+ 15%|█████████████▉                                                                               | 9000/60000 [7:43:57<140:19:23,  9.91s/it][INFO|trainer.py:2896] 2023-11-19 18:43:40,779 >> Saving model checkpoint to ./checkpoint-9000
+[INFO|configuration_utils.py:462] 2023-11-19 18:43:40,806 >> Configuration saved in ./checkpoint-9000/config.json
+[INFO|configuration_utils.py:568] 2023-11-19 18:43:40,819 >> Configuration saved in ./checkpoint-9000/generation_config.json
+[2023-11-19 18:44:31,177] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step9000 is about to be saved!
+[2023-11-19 18:44:31,206] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-9000/global_step9000/mp_rank_00_model_states.pt
+[2023-11-19 18:44:31,207] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-9000/global_step9000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-19 18:44:31,150 >> Model weights saved in ./checkpoint-9000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-19 18:44:31,155 >> Feature extractor saved in ./checkpoint-9000/preprocessor_config.json
+[2023-11-19 18:44:39,043] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-9000/global_step9000/mp_rank_00_model_states.pt.
+[2023-11-19 18:44:39,053] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-9000/global_step9000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-19 18:45:00,369] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-9000/global_step9000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-19 18:45:00,387] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-9000/global_step9000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-19 18:45:00,387] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step9000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-19 18:46:02,212 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|█████████████▉                                                                               | 9020/60000 [7:50:14<148:53:42, 10.51s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████                                                                               | 9039/60000 [7:53:29<143:37:22, 10.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████                                                                               | 9059/60000 [7:57:24<150:38:34, 10.65s/it]
+
+
+

wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.127, "train/learning_rate": 2.641677966101695e-06, "train/epoch": 0.13, "train/global_step": 8060, "_timestamp": 1700402241.9787862, "_runtime": 17892.944699287415, "_step": 53, "eval/loss": 0.1519775390625, "eval/wer": 9.046362608367886, "eval/runtime": 612.6298, "eval/samples_per_second": 2.781, "eval/steps_per_second": 0.695}
+{"train/loss": 0.0721, "train/learning_rate": 2.5909830508474577e-06, "train/epoch": 0.15, "train/global_step": 9060, "_timestamp": 1700413010.009219, "_runtime": 28660.975131988525, "_step": 104, "eval/loss": 0.1575927734375, "eval/wer": 8.839050131926122, "eval/runtime": 625.1584, "eval/samples_per_second": 2.726, "eval/steps_per_second": 0.681}