Training in progress, step 100

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9639db61fa3afa415f325f17778c13e62d2c06417eec9a534cbead6971678b1
+oid sha256:f120070c7b26a742434301fe7452b44398b0d4dcf7728cf4d3589fe871d25ac2
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:763110499dd02a8fe0814cb21c64756895e2d1f54f80087c73e22d0d5b992702
-size 6110
+oid sha256:70df2f9dd59a889eeb1b58377dad0394a42d84a5baa7c62dc106717a86339040
+size 6317

wandb/debug-internal.log

@@ -132,3 +132,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 07:42:35,367 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 07:42:35,369 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 07:42:35,942 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 07:42:37,943 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 07:42:40,060 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:42:45,017 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 07:42:46,018 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:42:46,035 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 07:42:46,036 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 07:42:46,039 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 07:42:51,081 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:42:56,082 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:43:00,593 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 07:43:00,594 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 07:43:00,594 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:43:00,595 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:43:00,953 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 07:43:01,036 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 07:43:01,037 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 07:43:01,039 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 07:43:01,159 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -4,4 +4,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 4.5756, "train/grad_norm": 30.161609649658203, "train/learning_rate": 4e-06, "train/epoch": 0.01, "train/global_step": 80, "_timestamp": 1712907755.3654647, "_runtime": 110.85774660110474, "_step": 3}
+{"train/loss": 4.4352, "train/grad_norm": 40.131675720214844, "train/learning_rate": 4.9999999999999996e-06, "train/epoch": 0.01, "train/global_step": 100, "_timestamp": 1712907780.5925796, "_runtime": 136.08486151695251, "_step": 4}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -132,3 +132,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 07:42:35,367 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 07:42:35,369 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 07:42:35,942 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 07:42:37,943 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 07:42:40,060 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:42:45,017 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 07:42:46,018 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:42:46,035 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 07:42:46,036 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 07:42:46,039 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 07:42:51,081 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:42:56,082 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 07:43:00,593 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 07:43:00,594 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 07:43:00,594 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 07:43:00,595 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 07:43:00,953 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 07:43:01,036 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 07:43:01,037 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 07:43:01,039 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 07:43:01,159 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report