Training in progress, step 2960

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae4eed7cd394c61aa3c6ecdd8052158a478064360315d029f17e32256eeee681
+oid sha256:38f3ea2e710f0123cd317966daf5b272ed615e4c14e337828c2d49a95e7b8e06
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10480f64b1ef70feeecb0dae3edf2a050cfa0b7299823ad2f4f3b47185de33c9
-size 36275
+oid sha256:de40781eb00de5bdf8551d130aa37f13f98e3fef388c451997820a31ec09baf3
+size 36486

wandb/debug-internal.log

@@ -2471,3 +2471,16 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:40:48,313 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:40:48,420 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 08:40:50,421 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:40:52,198 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:40:57,199 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:01,765 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:01,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:01,780 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:02,945 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:07,945 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:12,947 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:13,252 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:41:13,253 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:41:13,253 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:41:13,254 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:41:13,430 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.6372, "train/grad_norm": 36.61361312866211, "train/learning_rate": 2.8935890574033325e-05, "train/epoch": 0.24, "train/global_step": 2940, "_timestamp": 1712911248.310622, "_runtime": 3603.8029038906097, "_step": 146}
+{"train/loss": 3.5737, "train/grad_norm": 24.437808990478516, "train/learning_rate": 2.8917845792248085e-05, "train/epoch": 0.24, "train/global_step": 2960, "_timestamp": 1712911273.251721, "_runtime": 3628.7440028190613, "_step": 147}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -2471,3 +2471,16 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 08:40:48,313 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 08:40:48,420 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 08:40:50,421 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 08:40:52,198 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:40:57,199 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:01,765 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 08:41:01,766 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 08:41:01,780 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 08:41:02,945 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:07,945 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:12,947 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 08:41:13,252 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 08:41:13,253 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 08:41:13,253 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 08:41:13,254 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 08:41:13,430 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json