Training in progress, step 5820

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4f6bee707bd16994812f4ab6abe0b76396f1fcd5a795587aa16fd0375503283
+oid sha256:8f2d12c00c3910f5c8aadf1c3043d575f58f0e94d4e1904af9295d91088e5ba8
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2eab0a947d62fc249b8e4cfbe70298fd00b7cd7dc3828a535fd5885c222f4cb
-size 66448
+oid sha256:fd24b335c78ea5de3b8fc5fbfbf721f5af100dc189376f419a417d8e805b6e5b
+size 66659

wandb/debug-internal.log

@@ -4812,3 +4812,19 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:39:14,873 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 09:39:15,120 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
 2024-04-12 09:39:15,273 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:39:17,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:17,401 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:17,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:17,874 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:39:20,531 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:25,532 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:30,533 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:32,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:32,400 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:32,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:36,485 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:37,316 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:39:37,317 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:39:37,317 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:39:37,318 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:39:37,881 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -576,4 +576,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5436, "train/grad_norm": 59.62504959106445, "train/learning_rate": 2.4988728375175216e-05, "train/epoch": 0.46, "train/global_step": 5800, "_timestamp": 1712914754.3635612, "_runtime": 7109.855843067169, "_step": 289}
+{"train/loss": 3.561, "train/grad_norm": 40.504127502441406, "train/learning_rate": 2.495243304591236e-05, "train/epoch": 0.47, "train/global_step": 5820, "_timestamp": 1712914777.3155742, "_runtime": 7132.807856082916, "_step": 290}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -4812,3 +4812,19 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:39:14,873 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 09:39:15,120 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
 2024-04-12 09:39:15,273 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:39:17,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:17,401 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:17,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:17,874 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:39:20,531 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:25,532 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:30,533 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:32,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:32,400 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:32,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:36,485 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:37,316 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:39:37,317 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:39:37,317 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:39:37,318 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:39:37,881 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json