Training in progress, step 5800

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:283dd7cd18659a3c9d3097777e6a3109d1270cd6b8e6372800d079fa0c6abec5
+oid sha256:d4f6bee707bd16994812f4ab6abe0b76396f1fcd5a795587aa16fd0375503283
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2f2968e847c58abcbc29632045825ecf132a3b36c5554f9a8958652257c9ba
-size 66237
+oid sha256:f2eab0a947d62fc249b8e4cfbe70298fd00b7cd7dc3828a535fd5885c222f4cb
+size 66448

wandb/debug-internal.log

@@ -4797,3 +4797,18 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:38:50,421 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 09:38:50,422 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 09:38:50,863 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:38:53,865 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:38:54,150 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:38:59,151 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:02,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:02,400 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:02,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:04,492 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:09,493 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:14,364 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:39:14,365 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:39:14,365 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:39:14,366 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:39:14,873 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:39:15,120 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:15,273 DEBUG   SenderThread:485 [sender.py:send():379] send: stats

wandb/run-20240412_074044-widovcn3/files/output.log

@@ -574,4 +574,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5194, "train/grad_norm": 92.71965789794922, "train/learning_rate": 2.5024918927404005e-05, "train/epoch": 0.46, "train/global_step": 5780, "_timestamp": 1712914730.4198887, "_runtime": 7085.912170648575, "_step": 288}
+{"train/loss": 3.5436, "train/grad_norm": 59.62504959106445, "train/learning_rate": 2.4988728375175216e-05, "train/epoch": 0.46, "train/global_step": 5800, "_timestamp": 1712914754.3635612, "_runtime": 7109.855843067169, "_step": 289}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -4797,3 +4797,18 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 09:38:50,421 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 09:38:50,422 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 09:38:50,863 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:38:53,865 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 09:38:54,150 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:38:59,151 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:02,128 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 09:39:02,400 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 09:39:02,401 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 09:39:04,492 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:09,493 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:14,364 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 09:39:14,365 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 09:39:14,365 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 09:39:14,366 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 09:39:14,873 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 09:39:15,120 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 09:39:15,273 DEBUG   SenderThread:485 [sender.py:send():379] send: stats