diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -922,3 +922,1047 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 2024-04-12 08:02:16,290 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages 2024-04-12 08:02:16,469 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json 2024-04-12 08:02:17,979 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:19,471 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:02:22,980 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:27,981 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:31,288 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:02:31,288 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:02:31,329 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:02:33,409 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:38,410 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:39,529 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:02:39,531 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:02:39,531 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:02:39,531 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:02:40,479 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:02:41,479 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:02:44,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:45,064 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:02:46,288 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:02:46,289 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:02:46,329 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:02:49,362 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:54,363 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:02:59,364 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:01,288 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:03:01,289 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:03:01,329 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:03:05,330 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:07,184 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:03:07,185 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:03:07,186 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:03:07,186 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:03:07,490 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:03:09,491 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:03:10,947 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:15,065 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:03:16,066 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:16,288 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:03:16,288 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:03:16,291 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:03:21,356 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:26,357 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:30,601 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:03:30,602 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:03:30,602 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:03:30,603 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:03:31,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:03:31,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:03:31,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:03:31,385 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:31,501 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:03:33,502 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:03:36,386 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:41,387 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:45,066 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:03:46,307 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:03:46,308 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:03:46,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:03:47,354 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:51,370 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:03:51,372 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:03:51,372 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:03:51,372 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:03:51,510 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:03:53,129 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:03:53,510 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:03:58,130 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:01,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:04:01,308 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:04:01,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:04:03,360 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:08,361 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:13,362 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:15,067 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:04:16,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:04:16,309 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:04:16,350 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:04:16,758 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:04:16,760 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:04:16,760 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:04:16,761 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:04:17,520 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:04:18,510 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:19,521 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:04:23,511 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:28,512 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:31,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:04:31,309 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:04:31,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:04:34,356 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:39,357 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:42,654 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:04:42,656 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:04:42,656 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:04:42,656 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:04:43,531 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:04:44,365 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:45,069 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:04:45,532 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:04:46,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:04:46,308 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:04:46,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:04:50,358 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:04:55,358 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:00,359 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:01,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:05:01,309 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:05:01,349 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:05:05,420 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:10,421 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:10,582 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:05:10,583 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:05:10,583 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:05:10,584 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:05:11,543 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:05:13,544 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:05:15,070 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:05:16,071 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:16,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:05:16,309 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:05:16,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:05:21,394 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:26,395 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:31,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:05:31,309 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:05:31,335 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:05:32,335 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:32,484 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:05:32,485 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:05:32,486 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:05:32,486 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:05:32,553 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:05:35,554 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:05:37,389 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:42,390 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:45,071 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:05:46,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:05:46,309 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:05:46,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:05:47,420 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:52,421 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:05:55,258 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:05:55,260 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:05:55,260 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:05:55,261 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:05:55,562 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:05:57,563 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:05:58,003 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:01,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:06:01,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:06:01,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:06:03,389 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:08,390 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:13,391 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:15,077 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:06:16,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:06:16,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:06:16,350 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:06:18,427 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:20,569 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:06:20,570 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:06:20,571 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:06:20,571 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:06:20,574 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:06:23,576 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:06:24,347 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:29,348 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:31,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:06:31,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:06:31,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:06:34,356 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:39,357 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:44,358 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:45,073 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:06:46,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:06:46,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:06:46,311 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:06:47,067 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:06:47,069 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:06:47,069 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:06:47,071 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:06:47,586 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:06:49,586 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:06:49,816 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:54,817 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:06:59,818 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:01,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:07:01,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:07:01,350 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:07:05,429 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:10,429 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:12,553 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:07:12,555 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:07:12,556 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:07:12,556 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:07:12,596 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:07:15,075 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:07:15,597 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:07:16,075 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:16,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:07:16,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:07:16,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:07:21,416 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:26,417 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:31,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:07:31,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:07:31,351 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:07:32,393 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:37,394 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:37,530 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:07:37,531 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:07:37,531 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:07:37,534 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:07:37,605 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:07:39,606 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:07:43,266 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:45,076 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:07:46,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:07:46,310 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:07:46,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:07:48,431 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:53,432 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:07:58,433 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:01,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:08:01,311 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:08:01,344 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:08:03,838 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:08:03,839 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:08:03,840 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:08:03,840 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:03,840 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:08:04,615 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:08:05,616 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:08:09,549 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:14,550 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:15,077 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:08:16,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:08:16,311 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:08:16,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:08:20,406 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:25,407 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:26,840 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:08:26,841 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:08:26,842 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:08:26,842 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:08:27,624 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:08:29,625 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:08:30,589 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:31,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:08:31,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:08:31,312 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:08:36,367 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:41,368 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:45,078 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:08:46,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:08:46,311 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:08:46,343 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:08:47,343 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:52,339 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:08:52,340 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:08:52,340 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:08:52,341 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:08:52,633 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:08:53,047 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:08:53,634 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:08:58,048 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:01,310 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:09:01,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:09:01,312 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:09:03,441 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:08,442 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:13,443 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:15,080 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:09:16,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:09:16,312 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:09:16,352 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:09:16,855 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:09:16,856 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:09:16,856 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:09:16,857 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:09:17,643 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:09:18,845 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:19,644 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:09:23,846 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:28,847 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:31,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:09:31,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:09:31,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:09:34,363 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:37,604 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:09:37,606 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:09:37,606 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:09:37,606 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:09:37,652 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:09:39,429 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:39,653 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:09:44,429 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:45,081 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:09:46,311 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:09:46,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:09:46,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:09:49,432 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:54,433 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:09:58,710 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:09:58,711 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:09:58,712 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:09:58,712 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:09:59,661 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:10:00,431 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:01,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:10:01,312 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:10:01,315 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:10:01,662 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:10:06,370 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:11,371 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:15,081 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:10:16,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:10:16,312 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:10:16,352 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:10:16,440 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:21,441 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:22,705 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:10:22,706 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:10:22,707 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:10:22,707 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:10:23,670 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:10:25,671 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:10:27,429 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:31,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:10:31,314 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:10:31,354 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:10:33,405 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:38,406 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:43,407 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:45,083 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:10:46,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:10:46,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:10:46,353 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:10:48,434 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:49,687 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:10:49,688 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:10:49,688 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:10:49,689 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:10:50,682 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:10:51,682 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:10:54,428 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:10:59,428 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:01,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:11:01,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:11:01,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:11:04,440 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:09,441 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:14,442 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:15,083 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:11:16,312 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:11:16,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:11:16,340 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:11:16,773 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:11:16,774 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:11:16,775 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:11:16,775 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:11:17,694 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:11:19,507 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:19,695 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:11:24,509 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:29,509 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:31,315 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:11:31,316 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:11:31,316 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:11:35,441 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:40,210 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:11:40,211 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:11:40,211 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:11:40,212 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:11:40,704 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:11:40,933 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:41,705 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:11:45,085 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:11:46,086 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:46,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:11:46,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:11:46,314 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:11:51,409 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:11:56,410 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:01,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:12:01,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:12:01,354 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:12:02,400 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:04,914 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:12:04,915 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:12:04,915 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:12:04,916 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:12:05,715 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:12:07,654 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:07,716 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:12:12,655 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:15,086 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:12:16,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:12:16,313 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:12:16,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:12:18,435 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:23,436 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:28,437 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:31,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:12:31,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:12:31,314 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:12:31,374 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:12:31,375 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:12:31,376 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:12:31,376 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:12:31,726 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:12:33,729 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:12:34,257 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:39,258 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:44,259 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:45,087 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:12:46,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:12:46,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:12:46,314 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:12:49,392 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:54,393 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:12:55,097 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:12:55,099 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:12:55,099 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:12:55,099 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:12:55,738 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:12:58,739 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:12:59,832 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:01,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:13:01,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:13:01,314 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:13:05,357 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:10,358 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:15,088 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:13:16,089 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:16,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:13:16,315 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:13:16,355 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:13:19,507 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:13:19,508 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:13:19,509 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:13:19,509 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:13:19,748 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:13:21,253 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:22,749 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:13:26,254 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:31,255 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:31,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:13:31,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:13:31,314 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:13:36,457 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:41,458 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:45,089 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:13:45,227 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:13:45,228 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:13:45,229 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:13:45,229 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:13:45,759 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:13:46,313 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:13:46,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:13:46,315 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:13:47,341 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:48,760 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:13:52,342 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:13:57,343 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:01,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:14:01,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:14:01,315 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:14:03,334 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:08,334 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:09,458 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:14:09,459 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:14:09,459 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:14:09,459 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:14:09,768 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:14:12,770 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:14:14,192 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:15,090 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:14:16,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:14:16,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:14:16,315 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:14:19,369 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:24,370 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:29,371 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:31,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:14:31,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:14:31,315 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:14:33,527 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:14:33,528 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:14:33,528 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:14:33,528 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:14:33,778 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:14:35,275 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:36,780 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:14:40,276 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:45,091 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:14:46,092 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:46,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:14:46,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:14:46,315 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:14:51,388 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:54,679 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:14:54,680 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:14:54,680 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:14:54,680 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:14:54,786 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:14:56,399 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:14:56,787 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:15:01,314 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:15:01,315 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:15:01,316 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:15:01,435 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:06,436 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:11,437 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:15,097 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:15:15,807 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:15:15,808 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:15:15,808 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:15:15,808 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:15:16,503 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:16,504 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:15:16,504 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:15:16,544 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:15:16,795 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:15:18,796 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:15:21,631 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:26,632 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:31,497 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:15:31,498 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:15:31,538 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:15:32,576 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:37,577 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:39,618 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:15:39,619 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:15:39,620 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:15:39,620 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:15:39,803 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:15:42,805 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:15:43,315 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:45,094 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:15:46,497 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:15:46,498 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:15:46,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:15:48,638 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:53,639 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:15:58,640 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:00,473 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:16:00,475 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:16:00,475 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:16:00,477 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:16:00,812 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:16:01,498 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:16:01,498 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:16:01,502 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:16:02,812 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:16:04,621 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:09,622 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:14,622 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:15,095 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:16:16,498 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:16:16,499 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:16:16,539 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:16:20,573 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:22,223 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:16:22,224 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:16:22,224 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:16:22,225 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:16:22,820 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:16:24,821 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:16:25,924 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:30,925 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:31,498 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:16:31,498 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:16:31,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:16:36,584 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:41,585 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:45,095 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:16:46,502 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:16:46,503 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:16:46,543 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:16:46,618 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:47,880 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:16:47,881 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:16:47,881 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:16:47,882 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:16:48,830 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:16:50,831 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:16:51,653 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:16:56,654 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:01,499 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:17:01,499 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:17:01,503 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:17:02,546 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:07,546 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:11,638 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:17:11,640 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:17:11,640 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:17:11,640 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:17:11,839 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:17:13,340 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:14,840 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:17:15,097 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:17:16,499 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:17:16,499 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:17:16,503 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:17:18,544 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:23,545 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:28,546 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:31,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:17:31,501 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:17:31,533 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:17:34,533 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:39,259 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:17:39,261 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:17:39,261 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:17:39,262 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:17:39,851 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:17:39,996 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:42,852 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:17:44,997 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:45,098 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:17:46,499 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:17:46,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:17:46,503 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:17:50,631 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:17:55,632 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:00,632 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:01,499 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:18:01,499 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:18:01,540 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:18:06,569 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:06,620 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:18:06,621 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:18:06,621 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:18:06,622 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:18:06,861 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:18:08,862 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:18:12,348 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:15,099 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:18:16,499 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:18:16,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:18:16,503 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:18:17,552 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:22,553 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:27,554 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:31,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:18:31,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:18:31,541 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:18:31,955 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:18:31,956 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:18:31,956 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:18:31,957 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:18:32,719 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:32,872 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:18:34,873 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:18:37,720 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:42,721 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:45,100 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:18:46,499 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:18:46,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:18:46,504 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:18:48,589 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:53,590 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:18:53,851 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:18:53,852 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:18:53,853 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:18:53,853 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:18:53,881 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:18:56,883 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:18:59,583 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:01,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:19:01,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:19:01,504 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:19:05,552 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:10,553 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:15,101 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:19:16,102 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:16,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:19:16,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:19:16,541 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:19:18,398 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:19:18,399 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:19:18,400 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:19:18,400 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:19:18,893 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:19:20,894 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:19:21,266 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:26,267 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:31,268 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:31,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:19:31,500 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:19:31,504 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:19:36,553 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:41,006 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:19:41,008 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:19:41,008 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:19:41,009 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:19:41,791 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:41,902 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:19:42,902 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:19:45,102 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:19:46,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:19:46,501 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:19:46,504 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:19:47,605 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:52,606 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:19:57,608 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:01,500 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:20:01,501 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:20:01,541 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:20:02,608 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:07,563 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:20:07,564 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:20:07,564 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:20:07,564 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:20:07,911 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:20:08,308 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:10,913 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:20:13,309 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:15,104 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:20:16,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:20:16,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:20:16,505 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:20:18,536 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:23,537 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:28,538 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:31,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:20:31,501 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:20:31,536 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:20:31,759 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:20:31,760 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:20:31,760 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:20:31,761 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:20:31,920 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:20:34,479 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:34,921 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:20:39,480 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:44,481 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:45,105 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:20:46,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:20:46,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:20:46,505 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:20:49,567 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:54,560 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:20:54,561 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:20:54,561 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:20:54,562 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:20:54,929 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:20:55,295 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:20:56,929 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:21:00,296 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:01,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:21:01,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:21:01,505 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:21:05,621 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:10,622 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:15,106 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:21:16,107 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:16,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:21:16,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:21:16,542 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:21:17,495 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:21:17,497 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:21:17,497 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:21:17,497 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:21:17,938 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:21:20,939 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:21:21,217 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:26,218 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:31,219 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:31,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:21:31,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:21:31,506 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:21:36,551 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:41,552 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:45,107 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:21:45,158 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:21:45,159 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:21:45,160 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:21:45,160 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:21:45,950 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:21:46,501 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:21:46,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:21:46,506 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:21:46,950 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:21:47,536 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:52,537 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:21:57,538 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:01,502 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:22:01,502 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:22:01,542 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:22:02,546 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:07,423 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:22:07,425 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:22:07,425 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:22:07,426 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:22:07,958 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:22:08,147 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:08,959 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:22:13,148 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:15,109 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:22:16,502 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:22:16,503 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:22:16,506 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:22:18,578 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:23,579 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:28,580 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:31,337 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:22:31,338 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:22:31,338 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:22:31,339 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:22:31,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:22:31,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:22:31,517 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:22:31,968 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:22:32,969 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:22:34,066 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:39,068 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:44,069 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:45,109 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:22:46,512 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:22:46,512 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:22:46,516 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:22:49,640 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:54,640 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:22:56,698 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:22:56,699 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:22:56,700 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:22:56,700 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:22:56,978 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:22:58,979 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:23:00,440 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:01,512 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:23:01,512 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:23:01,516 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:23:05,620 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:10,620 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:15,111 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:23:16,112 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:16,512 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:23:16,513 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:23:16,553 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:23:21,600 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:22,901 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:23:22,902 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:23:22,902 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:23:22,903 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:23:22,989 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:23:24,989 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:23:26,647 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:31,512 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:23:31,513 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:23:31,516 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:23:32,639 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:37,640 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:42,641 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:45,112 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:23:46,512 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:23:46,513 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:23:46,553 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:23:46,704 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:23:46,705 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:23:46,705 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:23:46,706 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:23:47,000 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:23:48,434 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:49,000 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:23:53,435 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:23:58,436 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:01,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:24:01,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:24:01,513 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:24:03,586 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:08,587 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:09,545 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:24:09,546 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:24:09,547 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:24:09,547 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:24:10,010 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:24:13,011 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:24:14,397 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:15,113 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:24:16,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:24:16,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:24:16,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:24:19,545 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:24,546 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:29,546 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:31,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:24:31,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:24:31,551 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:24:34,465 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:24:34,466 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:24:34,466 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:24:34,467 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:24:35,021 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:24:35,195 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:37,022 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:24:40,196 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:45,114 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:24:46,115 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:46,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:24:46,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:24:46,518 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:24:51,649 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:24:56,650 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:01,392 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:25:01,393 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:25:01,393 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:25:01,394 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:25:01,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:25:01,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:25:01,516 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:25:02,032 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:25:02,134 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:03,032 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:25:07,136 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:12,137 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:15,116 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:25:16,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:25:16,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:25:16,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:25:17,560 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:22,561 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:26,744 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:25:26,745 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:25:26,746 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:25:26,746 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:25:27,043 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:25:28,462 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:29,044 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:25:31,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:25:31,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:25:31,517 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:25:33,635 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:38,636 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:43,637 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:45,117 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:25:46,513 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:25:46,514 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:25:46,554 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:25:47,103 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:25:47,105 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:25:47,105 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:25:47,106 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:25:48,052 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:25:48,865 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:49,053 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:25:53,866 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:25:58,867 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:01,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:26:01,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:26:01,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:26:04,644 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:09,645 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:12,782 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:26:12,783 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:26:12,783 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:26:12,784 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:26:13,062 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:26:15,062 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:26:15,117 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:26:15,119 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:16,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:26:16,515 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:26:16,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:26:20,618 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:25,619 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:30,621 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:31,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:26:31,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:26:31,549 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:26:34,255 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:26:34,257 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:26:34,257 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:26:34,257 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:26:35,071 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:26:36,052 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:37,072 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:26:41,053 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:45,119 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:26:46,119 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:46,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:26:46,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:26:46,517 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:26:51,618 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:56,620 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:26:58,963 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:26:58,965 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:26:58,965 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:26:58,965 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:26:59,081 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:27:01,082 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:27:01,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:27:01,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:27:01,517 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:27:01,633 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:06,634 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:11,635 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:15,119 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:27:16,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:27:16,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:27:16,556 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:27:17,597 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:22,598 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:26,555 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:27:26,557 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:27:26,557 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:27:26,558 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:27:27,093 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:27:28,303 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:29,094 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:27:31,514 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:27:31,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:27:31,518 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:27:33,618 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:38,619 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:43,620 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:45,121 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:27:46,515 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:27:46,515 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:27:46,556 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:27:49,565 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:27:51,496 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:27:51,497 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:27:51,498 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:27:51,498 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:27:52,103 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:27:53,104 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log +2024-04-12 08:27:55,234 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:28:00,235 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:28:01,515 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:28:01,516 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status +2024-04-12 08:28:01,519 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:28:05,549 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:28:10,551 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:28:15,082 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history +2024-04-12 08:28:15,083 DEBUG SenderThread:485 [sender.py:send():379] send: history +2024-04-12 08:28:15,084 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: summary_record +2024-04-12 08:28:15,084 INFO SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-12 08:28:15,113 INFO Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json +2024-04-12 08:28:15,122 DEBUG SenderThread:485 [sender.py:send():379] send: stats +2024-04-12 08:28:15,787 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report +2024-04-12 08:28:16,515 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status +2024-04-12 08:28:16,516 DEBUG HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-12 08:28:16,517 DEBUG SenderThread:485 [sender.py:send_request():406] send_request: stop_status