diff --git "a/wandb/run-20240412_074044-widovcn3/logs/debug-internal.log" "b/wandb/run-20240412_074044-widovcn3/logs/debug-internal.log" --- "a/wandb/run-20240412_074044-widovcn3/logs/debug-internal.log" +++ "b/wandb/run-20240412_074044-widovcn3/logs/debug-internal.log" @@ -3367,3 +3367,1022 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 2024-04-12 09:03:07,994 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json 2024-04-12 09:03:08,705 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report 2024-04-12 09:03:09,995 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:03:13,706 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:15,197 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:03:16,920 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:03:17,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:03:17,162 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:03:19,266 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:24,267 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:29,268 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:31,920 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:03:32,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:03:32,162 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:03:33,018 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:03:33,019 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:03:33,019 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:03:33,020 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:03:34,005 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:03:34,905 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:36,006 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:03:39,906 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:44,907 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:45,197 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:03:46,920 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:03:47,161 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:03:47,162 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:03:50,248 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:03:55,249 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:00,123 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:04:00,124 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:04:00,124 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:04:00,125 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:04:00,864 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:01,015 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:04:01,920 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:04:02,015 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:04:02,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:04:02,162 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:04:06,265 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:11,266 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:15,198 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:04:16,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:04:17,161 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:17,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:04:17,162 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:04:22,222 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:26,634 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:04:26,635 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:04:26,636 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:04:26,636 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:04:27,025 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:04:27,352 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:30,027 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:04:31,920 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:04:32,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:04:32,163 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:04:33,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:38,294 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:43,295 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:45,200 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:04:46,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:04:47,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:04:47,162 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:04:48,271 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:04:48,272 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:04:48,273 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:04:48,273 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:04:49,027 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:49,033 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:04:52,035 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:04:54,028 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:04:59,029 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:01,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:05:02,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:05:02,163 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:05:04,570 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:09,571 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:14,053 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:05:14,054 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:05:14,055 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:05:14,055 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:05:14,830 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:15,044 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:05:15,201 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:05:16,044 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:05:16,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:05:17,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:05:17,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:05:20,218 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:25,219 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:30,220 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:31,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:05:32,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:05:32,165 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:05:35,397 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:36,590 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:05:36,591 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:05:36,591 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:05:36,591 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:05:37,053 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:05:40,055 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:05:41,319 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:45,201 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:05:46,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:05:47,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:47,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:05:47,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:05:52,298 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:05:57,299 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:01,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:06:02,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:06:02,165 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:06:02,403 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:06:02,404 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:06:02,404 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:06:02,405 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:02,405 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:06:03,064 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:06:06,067 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:06:08,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:13,162 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:15,203 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:06:16,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:06:17,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:06:17,165 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:06:18,275 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:23,276 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:26,213 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:06:26,214 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:06:26,214 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:06:26,215 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:06:27,078 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:06:28,078 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:06:28,921 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:31,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:06:32,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:06:32,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:06:34,303 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:39,304 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:44,305 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:45,204 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:06:46,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:06:47,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:06:47,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:06:49,351 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:06:49,352 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:06:49,352 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:06:49,353 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:06:49,353 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:06:50,089 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:06:52,090 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:06:55,132 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:00,133 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:01,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:07:02,165 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:07:02,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:07:05,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:10,294 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:12,932 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:07:12,934 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:07:12,934 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:07:12,935 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:07:13,097 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:07:15,204 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:07:16,099 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:07:16,205 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:16,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:07:17,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:07:17,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:07:21,287 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:26,288 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:31,289 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:31,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:07:32,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:07:32,166 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:07:34,686 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:07:34,687 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:07:34,687 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:07:34,688 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:07:35,106 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:07:36,406 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:38,108 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:07:41,407 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:45,206 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:07:46,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:07:47,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:47,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:07:47,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:07:52,249 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:57,251 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:07:59,508 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:07:59,509 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:07:59,509 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:07:59,510 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:08:00,115 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:08:01,922 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:08:02,116 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:08:02,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:08:02,167 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:08:02,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:07,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:12,294 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:15,207 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:08:16,923 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:08:17,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:08:17,167 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:08:18,284 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:23,285 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:25,669 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:08:25,670 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:08:25,670 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:08:25,670 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:08:26,124 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:08:28,125 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:08:28,457 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:31,923 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:08:32,166 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:08:32,167 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:08:34,225 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:39,226 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:44,227 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:45,212 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:08:46,923 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:08:47,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:08:47,167 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:08:49,248 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:49,857 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:08:49,858 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:08:49,858 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:08:49,859 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:08:50,135 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:08:52,135 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:08:54,584 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:08:59,585 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:01,923 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:09:02,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:09:02,167 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:09:05,219 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:10,220 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:14,503 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:09:14,504 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:09:14,504 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:09:14,504 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:09:15,145 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:09:15,209 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:09:15,225 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:16,145 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:09:16,923 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:09:17,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:09:17,167 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:09:20,288 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:25,289 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:30,289 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:31,923 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:09:32,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:09:32,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:09:36,232 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:38,946 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:09:38,948 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:09:38,948 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:09:38,948 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:09:39,154 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:09:42,114 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:42,155 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:09:45,210 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:09:46,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:09:47,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:47,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:09:47,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:09:52,277 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:09:57,278 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:01,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:10:02,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:10:02,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:10:03,274 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:04,947 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:10:04,948 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:10:04,948 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:10:04,950 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:10:05,165 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:10:08,166 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:10:08,686 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:13,687 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:15,211 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:10:16,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:10:17,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:10:17,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:10:19,216 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:24,217 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:29,218 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:29,468 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:10:29,469 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:10:29,469 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:10:29,469 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:10:30,176 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:10:31,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:10:32,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:10:32,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:10:32,177 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:10:34,305 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:39,306 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:44,307 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:45,212 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:10:46,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:10:47,167 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:10:47,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:10:50,253 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:53,741 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:10:53,742 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:10:53,742 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:10:53,743 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:10:54,185 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:10:55,488 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:10:56,186 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:11:00,489 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:01,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:11:02,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:11:02,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:11:06,264 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:11,265 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:15,213 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:11:16,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:11:17,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:17,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:11:17,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:11:19,713 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:11:19,714 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:11:19,714 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:11:19,715 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:11:20,195 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:11:22,195 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:11:22,673 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:27,674 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:31,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:11:32,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:11:32,168 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:11:33,223 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:38,224 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:43,225 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:44,663 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:11:44,664 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:11:44,664 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:11:44,665 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:11:45,205 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:11:45,214 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:11:46,206 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:11:46,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:11:47,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:11:47,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:11:49,193 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:54,194 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:11:59,195 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:01,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:12:02,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:12:02,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:12:05,022 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:10,023 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:11,144 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:12:11,145 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:12:11,146 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:12:11,146 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:12:11,216 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:12:14,218 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:12:15,215 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:12:15,217 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:16,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:12:17,169 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:12:17,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:12:20,295 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:25,297 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:30,298 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:31,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:12:32,168 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:12:32,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:12:32,367 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:12:32,368 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:12:32,368 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:12:32,369 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:12:33,226 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:12:34,227 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:12:36,096 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:41,097 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:45,216 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:12:46,217 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:46,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:12:47,169 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:12:47,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:12:51,223 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:56,224 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:12:57,451 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:12:57,452 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:12:57,453 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:12:57,453 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:12:58,236 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:13:00,236 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:13:01,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:13:02,169 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:02,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:13:02,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:13:07,215 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:12,216 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:15,218 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:13:16,926 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:13:17,169 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:13:17,169 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:13:17,296 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:22,297 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:23,645 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:13:23,646 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:13:23,647 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:13:23,647 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:13:24,246 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:13:26,247 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:13:27,443 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:31,932 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:13:32,169 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:13:32,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:13:33,253 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:38,251 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:43,252 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:45,219 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:13:46,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:13:47,169 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:13:47,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:13:48,298 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:48,482 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:13:48,484 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:13:48,484 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:13:48,484 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:13:49,262 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:13:50,262 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:13:54,243 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:13:59,244 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:01,930 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:14:02,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:14:02,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:14:05,234 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:10,235 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:14,315 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:14:14,316 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:14:14,316 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:14:14,317 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:14:15,220 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:14:15,280 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:14:16,221 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:16,281 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:14:16,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:14:17,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:14:17,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:14:21,281 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:26,281 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:31,282 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:31,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:14:32,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:14:32,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:14:36,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:40,877 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:14:40,878 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:14:40,878 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:14:40,878 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:14:41,292 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:14:41,580 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:42,292 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:14:45,221 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:14:46,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:14:47,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:47,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:14:47,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:14:52,290 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:14:57,291 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:01,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:15:02,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:15:02,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:15:03,264 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:05,029 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:15:05,031 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:15:05,031 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:15:05,031 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:15:05,301 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:15:08,302 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:15:08,783 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:13,784 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:15,222 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:15:16,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:15:17,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:15:17,170 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:15:19,236 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:24,237 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:29,055 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:15:29,056 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:15:29,056 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:15:29,057 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:15:29,309 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:15:29,749 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:30,310 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:15:31,931 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:15:32,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:15:32,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:15:35,220 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:40,221 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:45,222 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:45,223 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:15:46,932 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:15:47,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:15:47,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:15:50,302 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:53,784 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:15:53,786 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:15:53,786 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:15:53,786 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:15:54,318 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:15:55,525 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:15:56,319 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:16:00,526 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:01,932 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:16:02,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:16:02,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:16:06,236 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:11,237 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:15,224 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:16:16,244 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:16:16,245 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:16:16,246 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:16:16,246 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:16,247 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:16:16,326 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:16:16,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:16:17,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:16:17,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:16:18,327 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:16:21,296 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:26,297 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:31,298 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:31,948 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:16:32,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:16:32,172 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:16:37,258 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:40,757 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:16:40,758 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:16:40,758 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:16:40,759 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:16:41,336 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:16:42,336 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:16:42,481 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:45,225 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:16:46,948 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:16:47,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:16:47,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:16:48,295 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:53,296 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:16:58,297 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:01,948 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:17:02,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:17:02,171 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:17:04,230 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:09,231 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:09,378 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:17:09,379 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:17:09,379 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:17:09,379 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:17:10,348 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:17:12,348 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:17:15,124 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:15,226 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:17:16,949 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:17:17,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:17:17,172 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:17:20,216 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:25,217 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:30,218 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:31,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:17:32,171 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:17:32,172 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:17:34,333 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:17:34,334 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:17:34,334 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:17:34,335 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:17:34,357 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:17:36,046 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:36,357 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:17:41,047 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:45,232 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:17:46,233 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:46,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:17:47,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:17:47,172 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:17:52,227 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:17:57,228 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:00,679 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:18:00,680 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:18:00,681 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:18:00,681 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:18:01,366 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:18:01,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:18:02,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:18:02,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:18:02,240 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:02,367 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:18:07,241 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:12,242 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:15,228 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:18:16,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:18:17,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:18:17,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:18:17,286 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:22,287 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:26,743 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:18:26,744 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:18:26,744 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:18:26,746 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:18:27,376 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:18:27,482 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:28,376 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:18:31,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:18:32,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:18:32,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:18:33,269 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:38,270 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:43,271 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:45,230 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:18:46,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:18:47,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:18:47,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:18:48,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:51,236 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:18:51,237 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:18:51,238 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:18:51,238 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:18:51,384 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:18:52,385 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:18:54,011 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:18:59,012 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:01,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:19:02,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:19:02,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:19:04,244 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:09,244 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:14,246 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:15,231 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:19:15,497 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:19:15,498 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:19:15,499 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:19:15,499 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:19:16,393 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:19:16,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:19:17,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:19:17,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:19:18,394 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:19:19,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:24,294 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:29,295 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:31,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:19:32,172 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:19:32,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:19:34,303 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:38,753 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:19:38,754 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:19:38,754 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:19:38,755 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:19:39,400 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:19:39,543 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:40,400 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:19:44,544 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:45,231 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:19:46,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:19:47,173 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:19:47,173 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:19:50,275 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:19:55,276 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:00,277 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:01,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:20:02,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:20:02,174 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:20:02,518 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:20:02,519 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:20:02,520 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:20:02,520 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:20:03,410 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:20:04,410 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:20:06,231 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:11,232 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:15,232 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:20:16,233 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:16,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:20:17,173 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:20:17,174 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:20:22,228 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:27,230 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:29,052 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:20:29,054 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:20:29,054 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:20:29,054 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:20:29,420 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:20:30,420 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:20:31,951 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:20:32,173 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:20:32,174 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:20:32,273 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:37,274 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:42,274 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:45,233 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:20:46,952 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:20:47,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:20:47,174 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:20:48,202 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:53,203 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:20:53,472 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:20:53,474 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:20:53,474 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:20:53,474 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:20:54,430 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:20:56,430 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:20:58,221 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:01,952 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:21:02,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:21:02,174 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:21:04,219 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:09,220 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:14,220 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:15,234 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:21:15,533 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:21:15,534 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:21:15,534 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:21:15,534 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:21:16,437 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:21:16,952 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:21:17,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:21:17,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:21:18,438 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:21:19,231 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:24,232 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:29,232 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:31,952 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:21:32,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:21:32,174 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:21:34,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:37,735 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:21:37,737 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:21:37,737 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:21:37,737 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:21:38,445 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:21:39,469 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:40,446 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:21:44,470 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:45,235 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:21:46,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:21:47,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:21:47,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:21:50,215 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:21:55,216 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:00,217 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:01,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:22:02,072 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:22:02,074 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:22:02,074 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:22:02,074 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:22:02,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:22:02,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:22:02,456 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:22:04,457 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:22:05,570 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:10,571 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:15,237 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:22:16,238 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:16,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:22:17,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:22:17,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:22:22,208 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:26,555 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:22:26,557 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:22:26,557 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:22:26,557 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:22:27,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:27,465 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:22:28,465 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:22:31,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:22:32,174 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:22:32,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:22:33,246 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:38,247 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:43,248 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:45,238 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:22:46,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:22:47,175 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:22:47,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:22:48,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:52,149 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:22:52,151 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:22:52,151 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:22:52,151 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:22:52,474 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:22:53,885 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:22:55,475 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:22:58,886 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:01,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:23:02,175 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:23:02,175 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:23:04,269 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:09,270 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:14,271 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:15,239 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:23:16,868 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:23:16,869 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:23:16,870 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:23:16,870 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:23:16,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:23:17,329 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:23:17,330 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:23:17,484 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:23:19,485 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:23:19,603 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:24,604 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:29,605 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:31,953 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:23:32,291 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:23:32,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:23:35,329 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:37,613 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:23:37,614 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:23:37,614 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:23:37,614 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:23:38,492 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:23:39,492 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:23:40,365 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:45,240 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:23:46,241 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:46,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:23:47,291 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:23:47,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:23:51,431 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:23:56,432 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:01,433 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:01,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:24:02,291 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:24:02,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:24:03,796 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:24:03,797 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:24:03,798 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:24:03,798 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:24:04,504 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:24:06,532 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:07,505 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:24:11,533 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:15,242 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:24:16,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:24:17,243 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:17,291 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:24:17,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:24:22,322 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:27,323 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:28,604 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:24:28,605 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:24:28,605 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:24:28,606 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:24:29,513 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:24:31,514 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:24:31,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:24:32,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:24:32,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:24:32,422 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:37,422 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:42,424 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:45,243 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:24:46,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:24:47,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:24:47,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:24:48,403 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:53,404 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:24:54,651 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:24:54,652 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:24:54,652 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:24:54,653 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:24:55,523 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:24:57,524 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:24:59,363 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:01,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:25:02,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:25:02,292 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:25:04,403 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:09,405 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:14,406 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:15,244 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:25:16,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:25:17,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:25:17,294 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:25:19,431 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:19,873 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:25:19,874 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:25:19,875 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:25:19,875 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:25:20,534 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:25:23,535 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:25:24,615 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:29,616 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:31,954 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:25:32,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:25:32,293 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:25:35,349 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:40,349 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:45,245 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:25:45,631 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:25:45,632 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:25:45,632 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:25:45,632 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:45,633 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:25:46,543 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:25:46,956 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:25:47,292 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:25:47,293 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:25:47,544 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:25:51,384 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:25:56,385 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:01,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:01,956 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:26:02,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:26:02,293 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:26:06,411 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:09,413 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:26:09,414 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:26:09,414 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:26:09,414 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:26:09,554 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:26:11,555 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:26:12,148 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:15,246 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:26:16,956 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:26:17,247 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:17,293 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:26:17,293 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:26:22,347 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:27,348 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:31,952 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:26:31,953 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:26:31,953 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:26:31,954 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:26:31,976 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:26:32,430 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:32,431 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:26:32,431 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:26:32,563 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:26:35,564 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:26:37,719 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:42,720 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:45,251 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:26:46,962 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:26:47,385 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:26:47,386 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:26:48,457 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:53,458 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:56,447 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:26:56,449 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:26:56,449 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:26:56,449 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:26:56,572 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:26:59,170 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:26:59,573 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:27:01,962 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:27:02,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:27:02,386 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:27:04,435 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:09,436 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:14,437 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:15,248 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:27:16,962 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:27:17,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:27:17,386 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:27:19,447 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:23,601 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:27:23,602 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:27:23,602 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:27:23,604 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:27:24,583 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:27:25,366 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:25,583 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:27:30,367 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:31,962 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:27:32,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:27:32,387 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:27:35,452 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:40,454 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:45,249 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:27:46,250 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:46,963 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:27:47,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:27:47,387 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:27:48,585 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:27:48,586 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:27:48,586 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:27:48,587 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:27:48,593 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:27:51,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:27:51,595 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:27:56,315 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:01,316 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:01,963 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:28:02,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:28:02,387 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:28:06,467 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:11,468 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:11,950 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:28:11,951 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:28:11,952 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:28:11,954 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:28:12,604 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 09:28:13,604 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 09:28:15,250 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 09:28:16,963 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:28:17,251 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:17,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:28:17,387 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:28:22,486 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:27,487 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:31,963 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 09:28:32,387 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 09:28:32,388 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 09:28:32,509 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 09:28:36,108 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 09:28:36,109 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 09:28:36,109 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 09:28:36,109 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 09:28:36,614 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json