Training in progress, step 6400

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d0dc7626ddde701087507da47f6483bfaa8f5f8e188458a83f2674165d89960
+oid sha256:3af3f46a335a87c6b76ec18533625ed12e8e29961e53f6dc3be542d5a74f0002
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d23a3f51d2d54618cb7cfc30bd46242395d22687c3ca69f2d25e25ffcaad08af
-size 72567
+oid sha256:ef5e97b0d444f440620049b96f9113c116121d59826b24f431102d7c66376d51
+size 72778

wandb/debug-internal.log

@@ -5281,3 +5281,16 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:50:47,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
 2024-04-12 09:50:47,410 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
 2024-04-12 09:50:47,411 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:50:49,146 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:50:51,549 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:50:56,550 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:01,551 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:02,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:51:02,412 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:51:02,412 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:51:07,508 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:10,197 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:51:10,198 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:51:10,199 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:51:10,199 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:51:11,154 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -634,4 +634,8 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.4775, "train/grad_norm": 24.581098556518555, "train/learning_rate": 2.3895199510167793e-05, "train/epoch": 0.51, "train/global_step": 6380, "_timestamp": 1712915445.6659112, "_runtime": 7801.15819311142, "_step": 318}
+{"train/loss": 3.5718, "train/grad_norm": 22.933914184570312, "train/learning_rate": 2.385603559508884e-05, "train/epoch": 0.51, "train/global_step": 6400, "_timestamp": 1712915470.1969364, "_runtime": 7825.6892182827, "_step": 319}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -5281,3 +5281,16 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:50:47,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
 2024-04-12 09:50:47,410 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
 2024-04-12 09:50:47,411 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:50:49,146 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:50:51,549 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:50:56,550 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:01,551 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:02,136 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:51:02,412 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:51:02,412 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:51:07,508 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:51:10,197 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:51:10,198 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:51:10,199 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:51:10,199 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:51:11,154 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json