diff --git a/run.recover.sh b/run.recover.sh index cb22ddd..1910986 100755 --- a/run.recover.sh +++ b/run.recover.sh @@ -5,7 +5,7 @@ WANDB_ENTITY=NbAiLab WANDB_PROJECT=wav2vec2 python run_speech_recognition_ctc.py --dataset_config_name="16K_mp3_bokmaal" \ --output_dir="./" \ --overwrite_output_dir \ - --num_train_epochs="18" \ + --num_train_epochs="11" \ --per_device_train_batch_size="16" \ --per_device_eval_batch_size="16" \ --gradient_accumulation_steps="2" \ diff --git a/runs/Feb07_09-59-22_dante/events.out.tfevents.1644224417.dante.3002729.0 b/runs/Feb07_09-59-22_dante/events.out.tfevents.1644224417.dante.3002729.0 index 97f808c..5e2ec46 100644 --- a/runs/Feb07_09-59-22_dante/events.out.tfevents.1644224417.dante.3002729.0 +++ b/runs/Feb07_09-59-22_dante/events.out.tfevents.1644224417.dante.3002729.0 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e780c588ec22c0b9a1d30c9386f6aeda92a9fe1f0196f71f6b6843e58d91b07a -size 30053 +oid sha256:5c8717fc73eeff12290f307d9fa63448656e1eb5636086356a037822bc5a4e31 +size 31156 diff --git a/special_tokens_map.json b/special_tokens_map.json index 9f67b88..ac9d0f5 100644 --- a/special_tokens_map.json +++ b/special_tokens_map.json @@ -1 +1 @@ -{"bos_token": "", "eos_token": "", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]} \ No newline at end of file +{"bos_token": "", "eos_token": "", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]} \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log index ebb26e4..6a98292 120000 --- a/wandb/debug-internal.log +++ b/wandb/debug-internal.log @@ -1 +1 @@ -run-20220207_100018-2lxyvusa/logs/debug-internal.log \ No newline at end of file +run-20220207_210259-2c5c0c93/logs/debug-internal.log \ No newline at end of file diff --git a/wandb/debug.log b/wandb/debug.log index 7da17c3..df8eb71 120000 --- a/wandb/debug.log +++ b/wandb/debug.log @@ -1 +1 @@ -run-20220207_100018-2lxyvusa/logs/debug.log \ No newline at end of file +run-20220207_210259-2c5c0c93/logs/debug.log \ No newline at end of file diff --git a/wandb/latest-run b/wandb/latest-run index 97a1950..fc546bc 120000 --- a/wandb/latest-run +++ b/wandb/latest-run @@ -1 +1 @@ -run-20220207_100018-2lxyvusa \ No newline at end of file +run-20220207_210259-2c5c0c93 \ No newline at end of file diff --git a/wandb/run-20220207_100018-2lxyvusa/files/config.yaml b/wandb/run-20220207_100018-2lxyvusa/files/config.yaml index d0570ee..1fe8516 100644 --- a/wandb/run-20220207_100018-2lxyvusa/files/config.yaml +++ b/wandb/run-20220207_100018-2lxyvusa/files/config.yaml @@ -4812,7 +4812,15 @@ _wandb: - 3 - 5 - 11 + 2: + - 1 + - 2 + - 3 + - 5 + - 11 3: + - 1 + - 7 - 13 4: 3.9.7 5: 0.12.9 diff --git a/wandb/run-20220207_100018-2lxyvusa/files/output.log b/wandb/run-20220207_100018-2lxyvusa/files/output.log index 6048d07..0dcde81 100644 --- a/wandb/run-20220207_100018-2lxyvusa/files/output.log +++ b/wandb/run-20220207_100018-2lxyvusa/files/output.log @@ -13252,3 +13252,582 @@ Deleting older checkpoint [checkpoint-4500] due to args.save_total_limit 0%| | 0/340 [00:00', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 16, 'eval_batch_size': 16} 2022-02-07 10:00:24,567 INFO MainThread:3002729 [wandb_watch.py:watch():43] Watching +2022-02-07 20:47:55,621 INFO MainThread:3002729 [wandb_run.py:_atexit_cleanup():1780] got exitcode: 1 +2022-02-07 20:47:55,624 INFO MainThread:3002729 [wandb_run.py:_restore():1752] restore +2022-02-07 20:47:57,786 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 1 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 35304 +} + +2022-02-07 20:47:57,948 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 1 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 35304 +} + +2022-02-07 20:47:58,686 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 1 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 35304 +} + +2022-02-07 20:47:59,624 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 4362102 +} + +2022-02-07 20:47:59,727 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 4362102 +} + +2022-02-07 20:47:59,829 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 4362102 +} + +2022-02-07 20:47:59,932 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 35304 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,034 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 672528 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,137 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 3126251 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,239 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,341 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,444 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,546 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,649 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,751 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:00,854 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:02,244 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} + +2022-02-07 20:48:02,828 INFO MainThread:3002729 [wandb_run.py:_wait_for_finish():1912] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 6 + other_count: 1 +} +pusher_stats { + uploaded_bytes: 4362102 + total_bytes: 4362102 +} +local_info { +} + +2022-02-07 20:48:04,247 INFO MainThread:3002729 [wandb_run.py:_append_history():2130] rendering history +2022-02-07 20:48:04,249 INFO MainThread:3002729 [wandb_run.py:_append_summary():2085] rendering summary +2022-02-07 20:48:04,250 INFO MainThread:3002729 [wandb_run.py:_append_files():2180] logging synced files diff --git a/wandb/run-20220207_100018-2lxyvusa/run-2lxyvusa.wandb b/wandb/run-20220207_100018-2lxyvusa/run-2lxyvusa.wandb index b40e4e4..6a5da6c 100644 --- a/wandb/run-20220207_100018-2lxyvusa/run-2lxyvusa.wandb +++ b/wandb/run-20220207_100018-2lxyvusa/run-2lxyvusa.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:40c91b6a5a818b62b0ac8783f2400e71497b70940ef2b53277003bbf21dc62aa -size 82619800 +oid sha256:78cbf3cf096803acfe3fae66ef4ff9de66cd71e7d1c9a466f75ae088e556059c +size 87079369