Training in progress, step 5840

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f2d12c00c3910f5c8aadf1c3043d575f58f0e94d4e1904af9295d91088e5ba8
+oid sha256:9a05d66290a553ce7e5848692f87b4d47dd70832cd730d3e060d68efb842c89b
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd24b335c78ea5de3b8fc5fbfbf721f5af100dc189376f419a417d8e805b6e5b
-size 66659
+oid sha256:fd58ebcbbc5d1bba93a83159196ededaaa242808dcd84ca3e65b20e286f962c8
+size 66870

wandb/debug-internal.log

@@ -4828,3 +4828,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:39:37,317 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 09:39:37,318 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 09:39:37,881 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:39:39,882 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:39:42,074 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:45,274 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:39:47,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:47,275 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:47,401 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:47,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:52,440 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:57,441 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:40:02,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:40:02,401 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:40:02,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:40:02,452 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:40:02,924 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:40:02,925 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:40:02,925 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:40:02,926 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:40:03,891 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -578,4 +578,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.561, "train/grad_norm": 40.504127502441406, "train/learning_rate": 2.495243304591236e-05, "train/epoch": 0.47, "train/global_step": 5820, "_timestamp": 1712914777.3155742, "_runtime": 7132.807856082916, "_step": 290}
+{"train/loss": 3.3979, "train/grad_norm": 43.67660140991211, "train/learning_rate": 2.4916033320336263e-05, "train/epoch": 0.47, "train/global_step": 5840, "_timestamp": 1712914802.9240758, "_runtime": 7158.416357755661, "_step": 291}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -4828,3 +4828,21 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:39:37,317 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 09:39:37,318 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 09:39:37,881 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:39:39,882 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:39:42,074 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:45,274 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
+2024-04-12 09:39:47,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:47,275 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:47,401 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:47,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:52,440 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:57,441 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:40:02,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:40:02,401 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:40:02,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:40:02,452 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:40:02,924 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:40:02,925 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:40:02,925 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:40:02,926 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:40:03,891 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json