diff --git "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" --- "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" +++ "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" @@ -8977,3 +8977,5836 @@ 2023-11-19 15:58:10,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status 2023-11-19 15:58:14,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log 2023-11-19 15:58:15,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:19,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 15:58:20,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:22,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:58:25,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:25,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:25,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:26,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:31,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:32,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:58:36,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:40,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:40,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:40,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:42,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:42,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:58:47,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:49,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 15:58:53,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:58:53,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:55,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:55,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:55,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:59,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:03,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:59:04,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:09,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:10,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:10,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:10,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:11,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:59:15,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:19,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 15:59:20,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:21,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:59:25,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:25,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:25,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:31,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:59:32,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:37,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:40,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:40,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:40,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:41,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:59:42,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:47,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:49,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 15:59:51,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 15:59:52,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:57,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:57,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:58,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:59,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:03,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:08,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:09,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:12,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:12,013 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:14,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:19,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:19,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:19,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:00:24,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:27,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:27,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:27,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:29,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:30,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:35,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:00:36,546 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:00:36,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:00:36,548 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:00:36,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:00:39,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:40,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:42,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:42,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:46,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:48,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:49,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:00:51,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:56,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:00:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:57,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:57,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:02,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:07,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:12,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:13,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:01:13,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:18,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:19,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:01:23,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:01:23,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:27,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:27,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:27,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:29,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:35,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:35,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:01:40,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:42,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:45,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:49,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:01:50,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:51,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:01:55,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:58,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:58,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:58,400 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:01,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:02:01,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:06,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:11,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:02:12,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:13,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:13,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:13,398 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:17,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:19,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:02:21,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:02:22,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:27,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:28,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:28,494 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:31,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:02:33,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:38,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:41,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:02:43,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:43,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:43,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:43,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:48,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:49,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:02:51,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:02:53,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:58,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:58,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:58,493 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:59,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:01,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:03:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:11,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:03:13,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:13,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:13,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:15,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:19,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:03:20,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:21,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:03:25,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:28,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:28,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:31,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:31,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:03:36,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:41,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:42,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:03:43,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:43,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:43,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:46,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:49,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:03:51,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:03:51,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:56,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:58,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:58,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:59,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:04:02,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:07,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:08,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:04:08,467 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:04:08,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:04:08,470 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:04:09,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:04:09,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:04:13,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:13,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:13,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:13,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:18,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:19,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:04:19,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:04:23,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:28,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:29,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:04:29,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:34,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:40,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:41,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:04:43,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:43,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:43,493 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:45,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:49,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:04:50,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:51,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:04:55,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:58,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:58,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:58,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:01,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:05:01,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:06,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:11,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:05:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:13,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:17,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:19,915 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:05:21,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:05:22,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:27,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:29,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:29,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:29,064 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:31,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:05:33,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:38,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:41,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:05:43,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:44,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:44,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:44,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:49,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:49,916 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:05:51,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:05:54,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:59,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:59,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:59,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:59,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:01,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:06:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:10,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:12,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:06:14,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:14,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:14,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:15,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:19,917 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:06:20,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:22,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:06:25,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:29,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:29,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:29,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:30,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:32,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:06:35,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:40,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:42,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:06:44,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:44,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:44,122 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:46,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:49,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:06:51,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:53,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:06:56,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:59,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:59,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:59,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:02,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:03,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:07:07,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:12,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:13,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:07:14,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:14,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:14,122 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:18,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:19,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:07:23,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:07:23,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:28,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:30,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:30,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:31,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:07:31,638 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:07:31,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:07:31,640 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:07:32,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:07:33,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:07:34,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:39,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:43,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:07:45,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:45,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:45,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:45,574 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:49,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:07:50,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:51,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:07:55,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:00,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:00,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:01,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:01,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:08:06,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:11,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:11,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:08:15,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:15,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:15,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:16,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:19,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:08:22,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:22,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:08:27,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:30,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:30,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:32,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:08:32,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:37,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:42,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:08:43,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:45,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:45,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:45,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:48,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:49,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:08:52,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:08:53,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:58,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:00,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:02,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:09:04,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:09,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:10,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:09:14,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:15,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:15,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:15,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:19,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:09:19,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:20,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:09:24,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:29,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:30,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:09:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:30,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:30,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:35,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:39,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:09:40,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:45,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:45,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:45,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:46,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:49,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:09:49,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:09:51,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:56,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:59,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:10:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:00,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:00,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:07,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:15,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:15,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:15,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:18,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:19,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:10:23,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:28,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:30,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:30,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:34,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:36,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:10:39,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:45,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:45,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:45,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:45,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:46,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:10:49,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:10:50,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:56,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:10:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:00,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:01,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:07,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:08,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:11:12,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:15,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:11:15,418 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:11:15,418 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:11:15,421 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:11:15,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:15,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:15,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:16,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:11:17,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:18,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:11:19,935 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:11:22,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:28,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:11:28,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:30,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:33,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:36,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:11:39,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:44,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:45,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:45,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:45,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:48,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:11:49,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:49,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:11:54,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:58,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:12:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:00,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:00,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:05,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:08,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:12:11,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:15,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:15,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:15,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:17,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:18,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:12:19,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:12:22,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:27,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:12:28,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:30,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:30,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:30,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:34,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:37,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:12:40,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:45,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:45,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:45,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:47,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:12:49,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:12:50,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:55,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:57,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:13:00,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:00,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:00,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:01,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:07,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:08,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:13:12,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:15,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:15,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:15,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:17,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:18,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:13:19,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:13:22,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:28,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:13:28,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:30,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:30,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:30,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:33,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:38,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:13:39,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:44,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:45,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:45,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:45,617 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:48,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:13:49,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:13:49,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:54,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:58,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:00,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:00,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:00,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:00,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:06,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:06,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:11,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:15,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:15,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:15,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:16,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:16,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:19,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:14:21,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:26,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:26,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:30,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:30,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:30,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:32,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:34,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:14:34,706 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:14:34,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:14:34,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:14:34,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:14:35,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:36,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:38,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:43,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:45,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:45,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:45,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:46,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:48,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:49,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:14:54,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:54,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:14:59,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:01,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:02,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:02,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:04,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:15:05,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:10,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:14,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:15:15,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:16,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:16,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:16,995 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:19,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:15:20,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:24,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:15:26,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:31,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:32,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:32,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:32,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:34,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:15:37,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:42,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:44,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:15:47,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:47,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:47,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:47,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:49,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:15:53,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:54,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:15:58,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:02,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:02,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:02,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:03,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:04,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:16:08,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:12,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:16:13,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:17,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:17,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:17,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:19,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:19,932 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:16:22,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:16:25,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:30,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:32,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:32,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:32,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:32,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:16:35,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:40,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:45,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:46,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:16:47,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:47,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:47,254 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:49,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:16:50,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:56,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:56,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:01,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:02,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:02,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:02,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:04,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:06,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:11,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:17,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:17,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:17,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:17,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:17,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:19,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:17:22,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:27,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:28,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:32,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:32,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:34,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:38,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:44,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:44,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:47,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:47,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:49,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:49,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:17:52,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:17:52,656 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:17:52,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:17:52,658 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:17:53,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:17:54,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:17:55,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:00,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:02,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:02,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:02,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:04,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:18:05,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:10,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:14,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:18:16,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:17,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:17,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:17,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:19,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:18:21,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:22,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:18:26,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:31,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:32,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:18:32,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:32,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:32,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:36,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:42,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:42,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:18:47,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:47,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:49,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:18:52,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:52,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:18:57,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:00,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:02,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:02,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:06,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:08,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:13,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:14,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:17,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:17,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:17,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:18,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:19,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:19:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:24,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:32,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:32,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:35,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:35,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:40,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:45,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:46,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:47,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:47,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:47,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:49,940 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:19:51,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:55,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:19:57,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:02,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:02,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:03,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:20:07,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:12,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:20:13,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:17,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:17,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:17,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:18,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:19,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:20:22,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:20:24,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:29,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:32,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:20:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:32,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:32,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:34,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:39,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:40,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:20:45,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:48,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:48,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:48,620 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:49,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:20:50,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:20:50,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:55,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:00,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:21:00,023 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:21:00,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:21:00,025 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:21:00,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:21:00,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:01,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:03,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:03,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:03,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:06,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:10,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:12,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:17,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:18,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:18,620 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:19,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:21:20,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:22,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:27,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:30,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:33,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:33,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:33,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:33,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:38,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:38,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:43,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:48,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:48,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:48,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:48,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:48,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:49,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:21:53,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:59,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:21:59,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:03,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:03,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:03,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:04,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:09,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:22:09,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:14,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:18,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:19,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:22:19,944 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:22:19,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:24,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:27,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:22:30,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:34,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:34,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:34,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:36,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:37,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:22:41,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:46,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:47,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:22:49,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:49,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:49,104 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:49,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:22:51,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:57,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:22:57,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:02,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:04,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:05,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:23:08,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:13,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:14,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:23:18,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:19,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:19,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:19,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:19,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:23:24,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:24,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:23:29,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:34,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:34,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:34,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:34,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:23:35,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:40,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:44,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:23:45,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:49,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:49,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:49,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:49,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:23:50,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:52,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:23:56,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:01,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:02,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:24:04,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:04,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:04,150 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:06,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:11,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:24:11,006 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:24:11,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:24:11,008 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:24:11,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:24:12,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:13,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:24:17,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:19,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:24:20,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:20,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:22,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:23,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:24:27,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:33,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:33,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:24:35,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:35,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:35,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:38,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:43,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:24:43,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:48,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:49,949 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:24:50,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:50,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:50,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:53,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:24:53,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:58,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:03,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:25:04,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:05,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:05,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:10,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:11,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:25:15,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:19,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:25:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:20,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:20,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:20,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:23,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:25:25,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:31,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:33,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:25:35,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:35,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:35,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:37,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:41,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:25:43,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:48,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:49,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:25:50,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:50,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:50,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:51,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:25:53,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:58,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:00,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:26:04,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:05,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:05,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:09,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:10,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:26:14,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:19,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:26:19,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:20,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:20,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:20,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:26:25,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:29,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:26:30,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:35,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:36,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:36,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:36,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:39,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:26:40,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:45,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:49,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:26:49,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:26:50,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:51,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:51,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:51,726 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:56,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:01,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:06,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:08,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:08,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:08,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:11,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:27:12,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:17,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:19,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:27:21,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:27:22,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:23,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:23,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:23,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:28,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:31,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:27:34,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:38,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:38,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:38,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:39,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:39,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:27:39,317 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:27:39,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:27:39,353 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:27:39,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:27:40,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:27:41,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:27:44,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:49,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:49,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:27:51,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:27:53,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:53,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:53,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:55,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:01,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:01,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:28:06,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:08,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:08,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:11,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:11,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:28:16,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:18,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:28:19,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:28:21,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:23,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:23,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:23,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:27,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:29,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:28:32,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:38,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:38,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:38,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:38,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:39,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:28:43,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:48,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:49,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:28:49,954 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:28:53,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:53,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:53,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:54,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:59,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:00,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:05,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:07,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:08,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:08,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:10,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:15,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:17,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:19,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:29:20,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:24,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:24,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:24,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:26,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:27,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:36,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:37,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:39,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:39,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:39,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:41,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:47,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:47,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:49,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:29:52,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:54,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:54,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:54,942 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:57,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:29:57,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:02,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:07,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:08,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:09,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:09,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:09,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:14,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:17,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:19,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:30:19,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:24,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:24,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:24,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:25,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:25,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:30,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:35,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:35,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:39,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:39,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:39,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:41,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:46,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:46,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:49,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:30:51,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:54,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:30:54,310 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:30:54,311 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:30:54,313 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:30:54,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:54,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:54,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:55,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:30:55,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:56,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:30:57,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:02,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:06,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:31:07,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:09,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:09,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:09,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:13,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:18,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:19,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:31:19,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:31:23,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:26,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:26,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:26,249 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:29,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:29,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:31:34,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:37,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:31:39,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:41,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:41,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:41,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:45,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:49,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:31:49,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:31:50,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:55,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:56,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:56,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:56,249 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:57,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:01,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:07,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:07,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:11,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:11,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:11,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:12,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:17,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:17,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:19,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:32:22,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:26,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:26,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:26,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:27,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:28,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:33,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:37,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:39,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:41,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:41,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:41,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:44,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:45,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:49,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:49,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:32:55,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:56,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:32:56,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:56,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:56,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:00,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:05,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:06,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:33:10,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:11,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:11,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:11,985 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:14,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:33:15,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:19,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:33:21,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:26,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:33:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:26,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:27,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:32,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:34,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:33:38,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:41,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:41,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:41,986 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:43,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:33:43,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:48,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:49,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:33:53,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:33:53,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:56,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:56,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:56,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:59,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:03,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:34:04,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:09,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:11,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:34:11,409 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:34:11,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:34:11,412 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:34:11,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:34:11,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:11,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:11,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:13,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:34:15,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:19,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:34:20,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:21,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:34:25,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:26,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:26,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:31,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:31,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:34:36,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:41,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:34:41,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:41,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:41,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:41,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:47,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:49,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:34:49,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:34:52,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:57,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:58,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:58,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:58,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:01,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:35:03,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:08,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:10,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:35:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:13,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:13,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:13,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:18,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:19,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:35:20,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:35:23,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:28,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:28,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:28,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:29,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:30,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:35:34,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:39,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:40,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:35:43,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:43,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:43,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:49,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:35:50,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:35:50,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:55,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:58,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:58,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:58,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:00,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:36:01,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:06,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:08,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:36:11,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:13,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:17,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:18,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:36:19,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:36:22,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:28,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:28,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:28,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:28,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:28,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:36:33,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:38,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:36:38,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:43,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:43,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:49,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:36:49,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:36:49,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:54,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:58,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:58,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:00,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:05,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:07,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:37:11,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:13,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:13,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:13,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:17,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:37:19,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:37:21,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:27,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:28,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:37:28,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:28,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:28,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:33,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:36,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:37:36,657 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:37:36,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:37:36,660 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:37:37,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:37:38,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:37:38,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:43,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:43,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:48,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:37:49,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:49,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:37:54,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:58,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:37:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:58,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:00,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:06,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:06,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:38:11,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:13,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:13,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:13,573 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:16,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:38:16,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:19,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:38:21,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:26,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:38:27,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:28,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:28,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:28,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:33,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:34,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:38:38,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:43,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:43,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:46,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:38:48,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:49,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:38:53,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:38:54,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:58,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:58,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:58,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:59,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:03,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:39:05,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:10,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:13,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:13,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:13,572 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:13,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:39:15,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:19,968 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:39:20,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:24,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:39:26,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:28,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:28,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:28,572 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:31,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:34,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:39:37,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:42,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:43,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:43,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:44,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:39:48,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:49,969 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:39:52,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:39:54,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:58,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:58,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:59,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:02,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:05,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:10,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:12,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:13,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:13,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:13,572 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:15,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:19,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:40:21,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:22,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:26,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:29,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:29,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:29,723 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:31,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:32,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:36,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:42,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:42,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:44,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:44,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:44,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:47,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:49,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:40:49,847 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:40:49,847 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:40:49,849 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:40:49,971 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:40:50,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:40:50,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:51,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:40:52,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:57,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:59,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:59,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:59,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:00,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:09,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:10,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:14,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:14,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:14,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:14,719 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:19,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:41:19,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:20,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:24,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:29,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:29,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:29,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:30,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:34,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:39,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:40,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:44,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:44,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:44,724 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:45,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:48,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:49,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:41:50,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:55,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:58,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:41:59,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:59,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:59,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:01,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:07,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:08,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:42:12,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:15,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:15,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:15,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:18,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:18,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:42:19,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:42:23,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:28,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:42:29,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:30,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:30,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:30,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:34,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:38,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:42:40,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:45,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:45,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:45,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:46,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:42:49,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:42:50,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:56,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:56,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:00,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:00,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:00,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:01,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:06,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:07,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:12,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:15,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:15,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:15,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:16,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:17,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:19,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:43:22,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:26,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:28,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:30,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:30,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:30,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:34,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:34,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:39,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:44,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:44,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:45,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:49,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:43:49,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:54,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:43:55,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:00,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:01,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:01,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:01,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:02,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:44:02,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:44:02,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:44:02,679 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:44:02,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:44:03,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:05,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:05,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:10,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:15,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:16,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:16,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:16,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:16,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:19,977 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:44:21,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:22,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:26,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:31,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:31,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:31,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:31,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:32,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:36,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:42,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:42,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:46,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:46,438 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:47,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:49,978 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:44:52,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:44:53,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:58,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:01,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:01,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:01,438 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:02,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:45:03,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:08,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:12,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:45:13,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:16,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:16,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:16,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:19,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:19,978 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:45:20,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:45:24,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:30,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:45:30,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:31,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:31,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:31,438 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:36,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:40,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:45:42,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:46,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:46,439 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:47,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:49,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:45:52,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:54,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:45:57,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:02,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:02,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:02,129 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:03,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:04,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:46:08,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:13,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:14,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:46:17,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:17,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:17,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:19,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:19,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:46:24,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:46:25,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:30,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:32,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:32,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:32,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:33,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:46:35,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:40,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:43,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:46:45,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:47,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:47,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:47,129 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:49,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:46:50,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:51,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:46:55,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:01,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:02,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:02,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:02,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:03,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:07,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:11,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:17,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:17,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:18,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:18,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:47:18,677 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:47:18,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:47:18,680 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:47:19,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:47:19,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:47:20,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:21,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:23,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:29,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:30,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:32,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:32,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:32,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:34,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:38,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:40,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:45,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:47,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:47,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:47,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:48,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:47:49,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:47:50,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:55,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:58,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:48:01,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:02,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:02,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:02,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:07,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:08,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:48:12,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:17,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:17,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:18,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:48:19,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:48:22,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:28,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:28,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:48:32,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:32,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:32,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:33,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:38,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:38,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:48:43,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:47,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:47,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:47,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:49,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:48:49,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:49,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:48:54,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:57,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:00,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:02,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:02,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:02,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:06,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:07,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:11,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:17,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:17,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:19,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:49:21,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:27,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:27,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:32,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:33,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:33,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:37,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:38,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:43,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:45,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:48,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:48,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:48,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:48,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:49,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:49:53,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:55,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:49:58,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:02,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:50:03,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:03,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:03,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:04,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:09,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:12,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:50:14,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:18,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:18,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:18,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:19,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:50:19,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:22,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:50:25,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:30,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:31,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:50:31,483 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:50:31,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:50:31,486 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:50:31,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:50:32,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:50:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:33,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:36,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:41,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:42,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:50:46,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:49,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:49,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:49,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:49,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:50:52,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:52,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:50:57,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:00,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:51:03,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:04,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:04,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:04,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:08,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:11,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:51:14,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:19,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:19,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:19,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:19,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:19,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:51:21,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:51:24,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:30,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:31,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:51:34,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:34,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:34,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:36,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:41,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:51:42,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:47,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:49,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:51:49,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:49,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:49,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:49,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:51:52,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:58,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:59,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:03,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:05,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:05,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:05,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:08,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:09,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:13,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:17,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:18,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:19,989 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:52:20,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:20,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:20,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:24,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:27,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:30,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:35,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:35,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:37,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:40,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:46,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:46,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:50,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:52:50,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:50,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:50,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:51,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:54,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:52:57,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:02,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:04,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:05,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:05,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:05,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:08,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:14,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:14,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:19,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:19,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:53:20,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:20,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:20,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:25,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:25,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:30,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:35,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:35,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:35,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:40,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:42,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:53:42,217 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:53:42,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:53:42,218 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:53:43,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:53:43,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:45,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:46,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:49,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:53:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:51,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:51,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:51,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:55,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:53:57,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:02,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:03,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:54:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:06,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:08,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:13,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:54:14,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:19,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:19,993 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:54:21,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:21,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:21,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:23,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:54:24,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:29,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:33,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:54:35,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:36,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:36,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:36,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:40,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:43,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:54:45,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:49,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:54:50,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:51,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:53,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:54:56,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:01,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:55:02,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:06,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:06,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:06,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:07,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:11,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:55:13,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:18,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:19,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:55:21,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:21,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:21,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:21,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:55:23,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:28,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:30,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:55:34,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:36,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:36,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:36,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:39,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:41,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:55:44,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:49,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:49,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:55:51,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:55:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:51,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:55,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:59,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:01,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:06,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:06,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:06,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:09,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:11,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:16,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:19,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:19,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:56:21,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:21,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:21,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:26,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:29,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:32,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:36,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:36,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:36,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:37,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:37,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:42,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:47,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:48,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:49,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:56:51,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:51,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:51,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:53,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:54,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:56:54,976 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 16:56:54,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:56:54,982 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:56:55,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 16:56:56,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:57,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:56:58,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:03,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:05,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:57:06,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:06,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:06,550 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:09,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:15,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:57:15,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:19,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:57:20,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:23,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:23,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:23,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:25,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:57:26,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:31,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:35,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:57:36,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:38,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:38,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:38,055 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:42,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:45,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:57:47,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:49,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:57:53,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:53,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:53,055 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:53,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:53,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:57:58,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:03,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:04,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:58:08,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:08,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:08,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:14,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:58:14,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:19,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:20,001 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:58:23,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:23,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:23,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:23,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:58:25,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:30,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:33,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:58:35,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:38,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:38,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:38,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:41,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:43,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:58:46,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:50,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:58:51,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:58:51,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:53,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:53,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:53,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:57,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:01,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:59:02,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:07,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:09,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:09,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:11,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:59:13,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:18,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:20,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:59:21,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:59:24,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:24,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:24,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:24,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:29,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:31,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:59:34,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:39,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:39,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:39,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:39,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:41,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:59:44,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:49,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:50,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 16:59:54,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:54,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:54,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:55,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 16:59:59,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:05,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:09,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:09,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:10,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:13,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:00:13,286 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:00:13,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:00:13,289 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:00:13,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:00:14,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:15,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:16,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:20,005 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:00:22,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:24,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:24,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:24,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:26,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:27,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:32,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:34,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:37,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:39,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:39,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:39,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:43,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:44,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:48,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:50,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:00:53,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:54,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:00:54,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:54,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:54,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:58,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:03,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:04,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:01:08,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:10,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:10,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:13,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:01:14,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:19,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:20,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:01:21,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:01:24,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:25,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:25,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:25,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:29,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:31,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:01:34,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:39,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:40,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:40,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:40,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:41,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:01:45,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:50,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:01:51,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:51,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:01:55,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:55,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:55,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:56,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:59,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:02:01,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:06,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:09,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:02:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:10,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:12,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:17,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:19,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:02:20,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:02:23,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:26,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:26,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:26,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:28,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:29,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:02:33,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:38,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:39,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:02:41,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:41,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:41,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:44,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:48,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:02:49,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:50,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:02:55,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:56,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:56,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:56,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:58,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:00,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:05,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:08,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:11,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:11,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:11,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:11,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:16,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:18,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:20,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:03:22,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:26,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:03:26,082 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:03:26,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:03:26,085 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:03:26,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:26,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:26,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:26,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:03:27,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:27,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:28,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:32,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:38,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:41,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:41,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:41,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:43,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:48,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:03:49,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:50,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:03:54,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:56,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:56,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:56,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:56,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:00,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:05,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:06,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:11,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:12,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:12,534 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:15,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:16,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:20,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:04:22,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:25,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:27,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:27,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:27,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:27,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:32,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:35,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:38,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:42,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:42,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:43,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:43,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:48,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:50,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:04:53,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:04:54,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:57,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:59,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:04,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:05:05,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:10,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:12,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:05:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:12,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:15,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:20,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:05:21,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:22,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:05:26,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:27,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:31,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:32,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:05:36,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:42,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:05:42,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:42,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:47,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:50,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:05:52,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:05:53,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:57,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:57,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:57,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:58,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:02,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:04,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:09,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:12,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:12,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:14,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:20,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:06:20,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:22,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:25,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:27,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:27,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:30,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:30,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:35,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:38,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:06:38,404 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:06:38,405 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:06:38,407 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:06:38,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:06:39,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:40,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:41,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:42,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:42,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:42,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:49,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:50,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:06:53,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:57,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:06:57,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:57,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:57,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:58,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:03,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:07,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:07:08,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:12,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:12,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:12,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:14,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:17,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:07:19,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:20,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:07:25,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:27,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:28,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:07:30,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:35,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:38,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:07:40,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:43,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:43,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:43,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:46,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:46,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:07:50,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:07:52,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:56,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:07:57,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:58,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:58,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:58,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:03,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:06,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:08:08,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:13,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:13,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:13,928 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:14,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:16,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:08:19,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:20,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:08:25,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:26,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:08:28,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:28,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:28,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:30,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:35,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:36,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:08:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:43,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:43,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:43,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:45,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:46,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:08:50,019 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:08:51,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:56,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:56,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:08:58,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:58,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:58,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:02,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:03,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:07,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:12,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:13,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:13,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:13,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:14,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:18,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:20,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:09:23,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:24,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:28,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:28,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:28,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:28,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:34,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:34,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:39,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:43,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:43,928 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:43,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:44,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:45,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:50,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:09:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:52,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:09:52,116 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:09:52,117 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:09:52,118 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:09:52,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:09:53,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:54,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:09:56,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:00,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:00,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:02,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:10:06,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:11,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:12,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:10:15,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:15,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:15,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:17,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:20,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:10:22,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:10:23,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:30,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:30,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:30,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:32,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:10:33,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:38,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:42,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:10:44,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:45,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:45,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:45,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:49,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:50,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:10:52,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:10:55,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:00,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:00,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:00,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:00,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:05,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:10,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:10,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:15,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:15,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:15,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:16,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:20,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:11:21,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:22,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:27,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:30,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:30,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:30,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:31,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:32,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:37,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:40,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:43,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:46,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:46,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:48,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:48,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:50,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:11:54,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:58,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:11:59,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:01,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:05,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:08,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:12:11,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:16,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:16,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:16,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:16,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:18,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:12:20,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:12:22,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:27,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:28,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:12:31,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:31,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:31,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:32,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:38,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:38,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:12:43,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:46,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:46,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:48,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:12:48,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:50,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:12:54,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:56,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:12:59,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:01,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:04,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:05,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:13:05,766 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:13:05,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:13:05,768 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:13:06,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:13:06,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:07,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:10,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:15,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:16,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:16,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:16,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:16,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:20,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:13:21,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:26,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:26,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:32,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:32,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:32,708 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:34,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:36,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:41,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:45,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:47,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:47,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:47,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:47,706 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:50,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:13:52,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:55,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:13:57,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:02,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:02,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:02,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:02,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:08,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:09,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:14:13,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:17,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:17,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:17,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:18,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:14:18,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:20,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:14:24,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:26,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:14:29,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:34,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:34,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:34,155 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:35,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:36,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:14:40,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:45,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:49,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:49,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:49,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:50,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:14:51,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:56,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:01,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:02,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:15:04,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:04,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:04,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:07,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:10,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:15:12,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:17,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:19,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:19,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:19,153 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:20,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:15:20,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:15:23,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:28,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:30,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:15:33,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:34,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:34,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:34,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:39,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:40,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:15:44,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:49,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:15:49,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:49,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:49,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:49,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:50,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:15:55,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:59,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:00,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:04,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:04,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:04,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:06,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:09,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:11,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:16,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:19,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:19,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:19,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:19,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:20,030 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:16:22,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:27,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:27,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:32,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:34,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:34,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:35,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:16:35,566 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:16:35,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:16:35,569 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:16:36,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:16:37,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:37,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:38,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:42,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:46,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:48,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:49,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:49,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:50,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:16:54,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:56,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:16:59,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:04,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:04,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:04,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:04,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:06,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:17:09,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:14,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:17:15,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:19,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:19,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:19,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:20,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:17:21,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:24,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:17:26,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:31,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:34,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:34,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:17:36,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:41,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:44,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:17:46,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:49,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:50,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:17:52,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:55,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:17:58,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:03,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:18:03,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:04,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:04,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:04,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:08,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:13,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:18:14,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:19,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:20,034 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:18:20,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:20,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:20,172 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:23,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:18:25,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:30,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:33,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:18:35,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:35,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:35,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:35,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:40,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:43,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:18:45,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:50,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:18:50,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:50,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:50,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:51,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:53,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:18:56,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:01,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:01,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:19:05,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:05,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:05,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:06,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:10,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:19:12,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:17,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:20,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:19:20,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:20,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:20,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:20,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:19:22,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:27,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:30,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:19:32,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:35,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:35,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:35,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:38,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:40,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:19:44,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:19:48,744 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:19:48,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:19:48,747 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:19:48,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:19:49,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:50,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:19:50,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:50,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:50,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:50,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:19:55,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:59,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:00,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:05,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:06,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:06,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:06,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:09,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:11,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:16,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:19,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:20,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:20:21,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:21,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:21,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:21,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:26,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:29,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:32,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:36,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:36,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:36,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:37,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:37,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:42,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:47,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:48,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:50,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:20:51,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:51,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:51,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:53,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:57,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:20:58,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:03,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:06,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:06,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:06,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:07,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:21:09,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:14,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:17,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:21:20,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:21:20,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:21,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:21,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:21,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:25,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:25,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:21:30,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:34,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:21:35,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:36,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:36,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:36,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:41,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:44,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:21:47,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:50,038 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:21:52,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:52,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:52,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:52,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:55,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:21:58,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:03,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:22:03,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:07,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:09,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:13,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:22:15,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:20,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:22:21,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:22,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:23,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:22:26,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:31,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:33,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:22:36,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:37,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:37,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:37,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:41,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:22:42,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:47,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:50,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:22:51,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:22:52,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:52,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:52,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:53,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:00,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:23:00,406 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:23:00,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:23:00,411 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:23:00,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:23:01,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:02,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:04,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:08,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:08,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:09,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:11,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:14,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:20,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:23:20,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:21,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:23,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:25,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:29,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:31,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:36,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:38,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:38,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:38,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:39,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:41,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:46,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:49,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:23:50,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:23:52,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:53,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:53,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:53,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:58,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:59,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:24:03,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:08,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:08,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:08,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:09,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:24:13,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:17,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:24:19,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:20,044 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:24:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:23,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:23,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:25,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:27,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:24:30,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:35,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:38,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:38,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:38,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:39,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:24:40,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:45,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:49,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:24:50,043 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:24:51,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:53,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:53,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:53,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:56,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:57,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:01,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:07,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:07,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:08,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:12,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:13,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:14,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:17,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:20,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:25:22,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:23,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:23,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:23,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:23,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:28,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:33,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:35,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:38,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:38,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:38,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:38,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:43,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:44,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:49,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:50,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:25:53,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:25:53,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:53,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:53,787 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:54,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:59,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:03,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:05,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:08,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:08,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:08,788 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:10,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:11,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:26:11,576 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:26:11,585 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:26:11,587 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:26:11,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:26:12,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:13,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:16,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:20,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:26:21,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:21,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:23,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:23,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:23,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:26,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:31,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:32,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:36,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:39,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:39,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:39,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:42,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:42,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:47,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:50,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:26:52,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:26:53,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:54,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:54,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:54,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:58,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:00,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:27:04,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:09,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:27:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:09,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:09,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:14,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:19,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:27:20,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:27:20,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:24,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:24,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:24,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:25,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:29,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:27:30,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:35,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:37,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:27:39,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:39,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:39,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:46,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:47,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:27:50,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:27:52,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:54,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:54,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:54,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:57,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:59,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:28:02,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:07,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:28:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:09,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:09,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:17,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:28:19,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:20,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:28:25,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:25,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:25,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:25,896 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:29,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:28:30,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:35,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:40,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:41,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:43,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:28:46,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:50,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:28:51,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:53,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:28:55,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:55,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:55,893 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:57,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:02,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:02,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:07,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:10,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:10,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:10,893 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:12,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:13,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:18,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:20,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:29:22,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:24,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:25,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:25,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:25,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:29,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:29,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:29:29,166 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:29:29,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:29:29,168 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:29:29,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:29:31,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:32,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:34,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:39,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:40,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:40,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:45,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:50,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:29:50,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:29:51,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:56,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:56,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:56,114 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:00,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:30:01,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:06,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:09,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:30:11,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:11,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:11,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:11,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:16,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:19,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:30:20,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:30:22,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:26,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:27,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:27,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:30:32,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:37,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:30:38,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:41,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:41,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:41,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:43,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:47,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:30:48,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:50,066 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:30:54,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:56,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:59,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:04,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:07,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:31:09,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:11,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:11,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:11,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:15,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:18,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:31:20,066 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:31:21,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:26,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:26,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:26,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:28,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:31:31,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:36,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:31:36,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:41,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:41,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:41,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:42,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:46,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:31:47,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:50,067 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:31:53,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:56,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:56,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:31:58,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:03,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:06,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:12,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:12,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:12,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:14,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:16,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:19,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:20,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:32:24,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:24,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:27,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:29,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:34,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:34,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:39,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:42,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:42,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:44,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:45,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:50,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:32:51,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:51,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:32:51,486 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:32:51,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:32:51,488 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:32:51,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:32:53,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:54,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:32:56,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:57,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:57,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:01,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:33:01,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:06,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:11,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:33:12,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:12,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:17,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:20,070 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:33:21,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:33:23,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:27,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:27,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:27,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:28,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:31,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:33:34,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:39,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:42,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:33:42,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:42,551 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:42,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:44,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:50,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:33:50,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:52,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:33:55,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:57,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:57,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:57,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:00,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:00,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:05,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:10,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:12,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:12,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:12,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:16,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:20,072 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:34:20,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:22,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:27,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:27,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:30,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:32,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:37,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:40,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:42,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:42,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:42,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:42,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:48,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:48,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:50,072 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:34:53,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:57,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:57,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:57,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:58,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:34:59,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:04,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:08,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:35:09,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:12,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:12,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:12,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:15,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:18,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:35:20,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:35:20,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:25,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:27,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:27,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:28,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:35:31,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:36,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:36,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:35:41,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:43,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:43,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:43,024 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:45,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:35:47,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:50,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:35:52,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:53,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:35:57,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:58,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:58,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:58,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:02,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:02,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:36:02,956 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:36:02,957 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:36:02,958 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:36:03,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:36:04,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:05,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:07,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:13,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:13,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:13,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:13,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:14,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:18,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:20,076 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:36:24,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:24,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:28,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:28,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:28,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:29,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:34,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:34,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:39,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:43,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:43,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:43,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:44,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:45,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:50,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:36:51,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:54,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:36:56,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:58,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:58,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:58,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:01,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:02,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:37:06,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:11,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:12,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:37:13,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:13,022 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:13,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:17,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:20,078 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:37:22,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:37:23,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:28,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:28,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:28,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:28,501 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:32,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:37:33,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:38,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:40,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:37:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:43,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:44,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:50,078 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:37:50,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:50,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:37:55,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:58,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:58,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:58,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:00,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:00,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:38:05,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:11,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:38:11,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:13,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:16,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:19,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:38:20,081 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:38:22,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:27,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:28,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:38:28,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:28,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:28,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:32,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:36,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:38:38,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:43,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:45,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:45,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:45,036 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:49,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:38:49,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:50,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:38:55,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:56,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:00,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:00,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:00,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:00,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:05,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:06,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:10,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:14,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:39:14,901 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:39:14,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:39:14,903 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:39:15,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:15,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:15,041 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:15,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:39:16,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:16,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:20,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:39:22,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:26,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:27,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:30,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:30,044 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:30,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:33,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:34,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:38,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:43,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:44,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:45,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:45,041 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:45,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:49,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:50,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:39:54,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:54,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:39:59,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:01,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:01,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:04,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:04,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:40:09,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:12,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:40:15,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:16,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:16,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:16,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:20,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:40:20,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:22,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:40:25,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:30,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:31,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:31,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:32,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:40:35,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:40,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:42,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:40:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:46,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:46,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:46,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:50,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:40:52,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:53,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:40:57,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:01,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:02,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:03,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:41:07,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:12,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:13,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:41:16,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:16,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:16,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:17,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:20,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:41:21,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:41:23,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:28,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:31,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:33,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:41:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:38,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:41,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:41:44,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:46,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:46,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:46,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:49,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:50,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:41:50,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:41:55,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:00,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:00,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:42:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:01,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:05,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:10,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:42:10,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:15,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:18,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:18,135 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:20,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:42:20,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:42:21,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:26,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:28,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:42:28,981 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:42:28,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:42:28,984 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:42:29,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:42:30,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:42:31,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:33,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:37,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:40,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:42:42,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:47,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:48,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:48,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:48,133 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:48,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:42:50,085 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:42:53,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:58,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:02,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:43:03,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:03,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:03,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:03,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:08,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:12,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:43:14,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:18,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:19,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:20,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:43:23,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:43:24,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:29,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:33,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:43:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:33,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:35,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:40,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:43,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:43:45,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:48,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:48,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:48,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:50,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:43:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:51,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:43:56,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:01,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:01,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:03,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:07,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:11,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:12,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:17,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:18,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:18,948 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:18,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:20,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:44:21,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:23,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:28,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:31,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:33,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:33,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:33,948 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:33,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:38,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:38,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:43,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:48,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:48,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:48,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:48,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:48,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:50,088 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:44:54,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:58,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:44:59,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:03,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:03,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:05,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:08,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:45:11,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:16,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:18,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:45:18,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:18,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:18,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:20,088 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:45:21,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:26,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:28,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:45:31,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:33,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:33,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:33,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:37,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:39,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:45:42,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:46,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:45:46,750 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:45:46,750 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:45:46,753 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:45:47,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:45:47,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:48,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:48,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:48,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:49,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:45:50,089 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:45:53,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:58,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:59,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:46:03,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:04,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:46:04,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:46:04,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:46:07,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:46:08,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:13,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:17,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:46:19,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:19,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:46:19,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:46:19,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:46:20,089 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:46:24,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:27,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:46:29,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:34,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:34,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:46:34,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:46:34,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:46:37,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:46:39,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:45,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:45,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:46:49,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:46:49,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:46:49,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:46:50,090 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:46:50,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:55,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:55,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:00,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:04,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:47:04,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:47:04,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:47:05,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:05,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:10,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:14,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:16,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:20,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:47:21,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:47:21,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:47:21,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:47:22,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:22,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:27,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:32,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:34,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:36,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:47:36,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:47:36,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:47:37,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:42,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:43,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:47,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:50,093 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:47:51,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:47:51,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:47:51,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:47:52,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:47:53,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:47:57,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:02,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:03,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:48:06,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:48:06,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:48:06,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:48:08,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:13,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:48:13,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:18,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:20,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:48:21,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:48:21,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:48:21,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:48:23,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:48:24,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:29,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:33,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:48:35,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:36,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:48:36,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:48:36,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:48:40,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:43,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:48:45,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:50,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:48:51,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:51,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:48:51,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:48:51,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:48:53,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:48:56,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:48:59,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:48:59,874 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:48:59,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:48:59,877 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:49:00,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:49:01,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:49:01,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:06,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:49:06,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:49:06,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:49:07,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:11,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:49:12,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:17,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:20,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:49:21,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:49:21,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:49:21,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:49:21,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:49:23,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:28,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:30,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:49:33,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:36,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:49:36,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:49:36,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:49:38,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:40,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:49:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:49,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:49:49,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:50,096 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:49:51,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:49:51,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:49:51,043 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:49:55,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:49:59,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:50:01,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:06,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:50:06,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:50:06,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:50:09,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:50:11,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:16,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:19,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:50:20,096 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:50:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:50:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:50:21,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:50:22,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:27,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:29,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:50:32,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:36,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:50:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:50:36,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:50:38,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:39,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:50:43,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:49,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:49,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:50:50,108 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:50:51,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:50:51,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:50:51,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:50:54,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:50:57,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:00,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:05,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:51:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:51:06,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:51:07,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:10,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:15,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:17,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:20,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:51:21,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:21,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:51:21,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:51:21,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:51:26,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:27,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:31,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:36,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:51:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:51:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:36,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:51:37,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:41,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:47,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:47,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:50,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:51:51,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:51:51,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:51:51,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:51:52,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:51:55,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:51:58,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:03,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:06,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:52:06,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:52:06,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:52:08,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:08,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:13,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:14,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:52:14,681 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:52:14,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:52:14,684 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:52:15,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:52:15,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:18,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:19,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:20,098 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:52:21,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:52:21,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:52:21,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:52:25,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:26,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:30,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:35,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:35,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:52:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:52:36,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:52:41,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:45,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:47,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:50,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:52:52,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:52:52,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:52:52,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:52:52,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:52:55,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:52:58,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:03,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:05,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:53:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:53:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:53:07,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:53:09,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:13,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:53:15,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:20,100 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:53:21,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:53:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:53:22,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:53:23,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:53:26,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:32,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:33,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:53:37,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:53:37,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:53:37,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:53:38,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:43,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:53:44,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:49,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:50,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:53:52,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:53:52,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:53:52,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:53:53,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:53:54,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:53:59,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:03,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:54:05,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:54:07,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:54:07,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:54:10,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:11,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:54:15,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:20,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:54:21,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:21,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:54:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:54:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:54:22,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:54:26,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:32,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:54:32,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:37,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:38,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:54:38,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:54:38,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:54:42,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:54:42,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:47,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:50,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:54:52,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:54:53,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:54:53,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:54:53,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:54:53,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:54:58,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:02,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:55:04,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:08,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:55:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:55:08,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:55:09,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:10,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:55:14,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:19,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:55:19,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:20,103 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:55:23,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:55:23,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:55:23,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:55:25,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:27,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:55:27,316 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:55:27,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:55:27,319 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:55:27,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:55:29,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:55:31,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:36,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:38,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:55:38,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:55:38,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:55:39,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:55:41,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:46,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:49,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:55:50,104 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:55:52,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:55:53,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:55:53,631 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:55:57,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:55:57,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:02,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:07,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:56:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:56:08,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:56:12,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:17,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:18,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:20,104 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:56:23,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:56:23,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:56:23,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:56:23,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:25,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:29,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:35,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:35,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:38,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:56:38,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:56:38,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:56:40,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:46,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:46,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:50,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:56:52,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:56:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:56:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:56:53,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:56:56,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:56:57,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:02,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:06,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:57:07,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:57:08,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:57:08,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:57:13,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:19,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:20,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:57:20,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:57:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:57:23,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:57:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:57:24,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:29,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:34,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:38,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:57:38,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:57:38,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:57:39,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:44,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:49,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:57:50,108 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:57:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:57:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:57:53,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:57:54,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:57:55,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:00,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:03,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:58:06,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:08,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:58:08,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:58:08,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:58:11,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:13,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:58:17,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:20,109 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:58:22,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:58:23,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:58:23,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:58:23,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:58:27,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:32,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:33,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:58:37,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:38,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:58:38,632 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:58:38,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:58:43,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:43,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:58:48,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:50,110 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:58:51,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:58:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:58:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:58:53,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:58:58,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:01,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:59:04,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:09,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:09,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:59:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:59:09,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:59:10,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:59:10,706 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 17:59:10,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:59:10,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:59:11,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 17:59:12,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:59:14,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:19,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:20,110 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:59:22,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:59:24,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:59:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:59:24,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:59:24,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:29,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:32,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:59:35,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:39,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:59:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:59:39,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:59:40,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:42,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:59:45,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:50,111 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 17:59:51,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:59:52,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 17:59:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:59:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:59:54,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:59:56,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:02,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:00:02,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:07,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:09,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:00:09,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:00:09,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:00:10,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:00:12,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:17,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:20,111 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:00:20,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:00:23,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:00:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:00:24,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:00:28,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:30,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:00:33,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:39,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:00:39,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:00:39,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:00:40,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:00:44,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:50,112 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:00:50,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:50,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:00:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:00:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:00:54,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:00:55,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:00:59,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:01,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:06,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:07,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:01:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:01:09,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:01:11,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:16,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:17,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:20,112 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:01:22,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:01:25,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:01:25,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:01:27,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:27,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:32,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:38,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:38,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:40,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:01:40,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:01:40,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:01:43,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:48,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:48,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:50,114 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:01:54,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:01:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:01:55,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:01:55,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:01:58,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:01:59,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:04,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:08,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:02:09,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:10,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:02:10,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:02:10,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:02:15,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:16,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:02:20,116 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:02:21,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:25,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:02:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:02:25,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:02:25,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:02:25,757 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:02:25,757 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:02:25,760 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:02:26,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:02:26,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:28,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:02:31,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:36,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:38,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:02:40,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:02:40,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:02:40,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:02:42,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:46,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:02:48,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:50,115 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:02:54,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:02:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:02:55,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:02:55,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:02:56,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:02:59,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:05,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:06,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:03:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:03:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:03:10,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:03:10,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:16,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:16,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:03:20,116 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:03:22,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:03:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:03:25,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:03:26,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:03:27,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:32,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:34,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:03:37,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:03:40,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:03:40,351 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:03:43,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:43,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:03:48,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:50,119 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:03:53,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:03:54,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:03:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:03:55,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:03:55,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:03:58,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:04,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:04:04,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:09,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:10,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:04:10,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:04:10,796 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:04:14,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:04:15,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:20,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:20,120 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:04:22,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:04:25,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:25,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:04:25,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:04:25,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:04:30,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:32,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:04:35,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:40,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:04:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:04:40,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:04:41,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:42,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:04:46,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:50,120 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:04:50,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:04:51,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:04:55,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:04:55,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:04:55,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:04:56,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:02,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:02,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:07,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:10,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:10,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:05:10,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:05:10,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:05:13,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:18,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:20,121 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:05:20,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:23,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:25,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:05:25,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:05:25,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:05:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:30,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:38,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:05:38,470 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:05:38,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:05:38,474 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:05:38,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:05:39,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:39,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:05:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:05:40,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:05:40,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:45,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:50,121 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:05:50,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:50,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:05:55,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:05:55,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:05:55,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:05:55,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:06:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:03,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:06:05,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:10,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:06:10,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:06:10,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:06:11,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:11,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:06:16,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:20,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:06:21,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:21,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:06:26,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:27,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:06:27,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:06:27,047 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:06:30,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:06:31,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:36,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:38,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:06:41,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:42,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:06:42,045 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:06:42,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:06:47,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:48,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:06:50,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:06:53,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:57,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:06:57,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:06:57,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:06:58,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:06:58,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:03,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:08,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:08,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:12,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:07:12,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:07:12,045 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:07:14,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:18,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:19,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:20,134 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:07:25,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:27,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:07:27,045 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:07:27,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:07:28,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:30,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:35,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:38,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:40,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:42,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:07:42,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:07:42,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:07:45,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:46,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:50,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:07:51,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:56,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:07:57,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:07:57,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:07:57,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:07:57,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:08:02,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:07,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:08:08,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:13,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:08:13,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:08:13,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:08:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:08:18,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:20,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:08:24,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:27,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:08:28,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:08:28,348 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:08:28,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:08:29,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:36,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:08:40,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:08:43,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:08:43,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:08:46,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:46,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:08:50,125 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:08:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:53,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:08:53,400 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:08:53,401 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:08:53,403 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:08:53,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:08:54,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:08:57,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:08:58,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:08:58,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:08:58,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:09:02,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:04,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:09:07,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:13,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:13,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:09:13,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:09:13,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:09:14,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:09:18,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:20,128 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:09:24,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:24,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:09:28,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:09:28,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:09:28,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:09:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:34,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:09:34,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:39,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:09:43,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:09:43,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:09:44,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:09:45,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:50,128 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:09:51,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:52,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:09:56,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:09:58,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:09:58,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:09:58,352 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:10:02,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:02,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:10:07,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:12,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:13,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:10:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:10:13,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:10:13,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:10:18,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:20,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:10:23,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:10:24,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:29,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:29,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:10:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:10:29,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:10:33,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:10:34,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:39,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:43,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:10:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:10:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:10:44,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:10:50,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:10:50,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:51,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:10:55,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:10:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:10:59,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:10:59,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:11:00,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:03,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:11:05,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:11,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:13,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:11:14,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:11:14,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:11:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:11:17,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:20,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:11:21,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:11:23,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:28,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:29,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:11:29,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:11:29,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:11:30,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:11:33,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:38,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:40,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:11:43,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:44,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:11:44,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:11:44,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:11:49,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:50,145 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:11:50,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:11:55,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:11:59,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:11:59,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:11:59,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:12:00,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:04,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:12:05,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:10,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:13,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:12:13,216 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:12:13,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:12:13,219 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:12:13,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:12:14,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:12:14,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:12:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:12:14,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:12:16,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:20,146 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:12:21,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:24,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:12:26,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:12:29,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:12:29,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:12:32,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:37,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:12:37,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:42,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:12:44,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:12:44,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:12:47,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:12:48,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:50,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:12:54,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:12:57,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:12:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:12:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:12:59,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:13:00,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:05,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:07,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:13:10,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:13:14,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:13:14,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:13:16,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:17,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:13:20,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:13:21,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:26,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:27,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:13:29,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:13:29,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:13:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:13:32,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:35,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:13:37,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:42,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:13:44,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:13:44,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:13:48,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:50,148 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:13:54,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:59,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:13:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:13:59,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:13:59,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:14:05,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:10,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:14,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:14:14,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:14:14,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:14:16,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:19,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:14:20,148 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:14:21,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:26,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:29,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:14:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:14:29,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:14:29,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:14:32,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:37,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:38,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:14:42,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:44,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:14:44,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:14:44,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:14:48,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:48,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:14:50,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:14:53,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:58,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:14:59,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:14:59,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:14:59,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:14:59,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:15:04,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:08,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:15:09,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:14,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:15,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:15:15,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:15:15,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:15:19,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:15:19,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:20,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:15:25,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:27,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:15:30,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:15:30,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:15:30,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:15:30,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:36,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:37,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:15:41,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:45,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:15:45,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:15:45,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:15:46,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:47,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:15:50,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:15:52,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:15:57,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:15:58,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:00,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:16:00,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:16:00,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:16:03,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:04,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:16:04,596 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:16:04,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:16:04,598 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:16:05,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:16:05,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:07,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:09,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:14,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:15,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:15,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:16:15,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:16:15,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:16:20,150 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:16:20,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:25,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:25,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:30,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:30,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:16:30,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:16:30,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:16:35,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:36,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:41,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:16:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:16:45,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:16:46,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:47,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:50,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:16:52,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:16:55,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:16:57,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:00,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:17:00,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:17:00,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:17:02,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:05,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:17:08,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:13,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:15,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:17:15,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:17:15,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:17:15,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:17:18,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:20,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:17:23,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:25,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:17:28,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:30,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:17:30,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:17:30,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:17:34,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:35,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:17:39,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:44,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:45,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:17:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:17:45,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:17:45,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:17:50,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:17:50,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:17:53,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:17:55,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:00,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:00,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:18:00,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:18:00,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:18:03,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:18:05,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:10,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:13,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:18:15,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:18:15,685 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:18:15,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:18:15,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:20,154 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:18:21,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:23,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:18:26,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:30,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:18:30,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:18:30,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:18:31,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:33,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:18:36,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:41,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:43,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:18:45,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:18:45,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:18:45,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:18:47,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:50,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:18:52,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:18:53,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:18:57,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:01,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:19:01,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:19:01,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:19:02,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:03,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:13,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:13,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:16,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:19:16,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:19:16,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:19:18,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:20,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:19:21,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:19:21,997 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:19:21,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:19:21,999 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:19:22,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:19:22,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:24,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:24,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:29,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:31,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:19:31,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:19:31,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:19:34,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:34,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:39,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:42,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:46,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:19:46,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:19:46,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:19:50,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:19:50,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:19:51,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:19:55,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:00,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:01,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:20:01,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:20:01,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:20:01,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:20:05,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:10,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:11,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:20:15,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:16,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:20:16,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:20:16,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:20:20,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:20:21,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:21,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:20:26,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:29,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:20:31,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:20:31,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:31,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:20:31,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:20:36,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:39,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:20:42,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:20:46,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:20:46,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:20:47,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:49,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:20:50,158 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:20:53,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:59,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:20:59,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:21:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:21:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:21:01,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:21:04,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:09,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:21:10,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:15,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:16,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:21:16,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:21:16,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:21:17,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:21:20,158 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:21:21,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:26,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:27,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:21:31,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:31,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:21:31,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:21:31,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:21:36,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:37,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:21:41,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:46,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:21:46,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:21:46,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:21:48,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:21:50,159 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:21:52,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:57,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:21:58,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:01,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:22:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:22:01,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:22:02,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:08,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:08,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:13,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:22:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:22:16,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:22:18,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:18,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:20,159 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:22:24,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:28,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:29,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:31,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:22:31,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:22:31,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:22:34,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:36,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:22:36,016 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:22:36,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:22:36,019 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:22:36,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:22:37,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:38,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:40,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:45,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:46,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:22:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:22:46,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:22:47,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:22:50,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:22:50,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:55,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:22:57,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:23:01,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:01,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:23:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:23:01,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:23:06,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:07,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:23:11,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:15,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:23:16,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:23:16,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:23:16,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:23:20,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:23:22,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:25,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:23:27,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:31,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:23:31,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:23:31,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:23:33,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:35,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:23:39,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:44,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:45,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:23:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:23:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:23:46,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:23:49,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:50,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:23:55,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:23:55,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:00,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:02,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:24:02,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:24:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:24:05,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:05,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:10,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:14,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:16,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:17,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:24:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:24:17,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:24:20,163 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:24:22,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:24,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:27,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:32,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:24:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:24:32,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:24:32,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:34,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:38,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:43,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:44,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:47,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:24:47,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:24:47,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:24:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:24:50,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:24:54,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:24:55,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:00,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:02,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:25:02,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:25:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:25:04,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:25:05,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:10,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:14,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:25:16,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:25:17,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:25:17,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:25:20,166 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:25:22,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:24,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:25:27,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:32,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:25:32,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:25:32,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:25:33,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:25:37,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:41,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:25:43,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:47,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:25:47,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:25:47,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:25:48,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:50,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:25:50,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:25:50,957 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:25:50,958 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:25:50,960 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:25:51,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:25:51,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:25:53,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:25:58,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:01,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:26:02,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:26:02,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:26:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:26:04,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:09,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:11,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:26:14,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:26:17,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:26:17,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:26:20,167 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:26:20,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:25,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:30,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:32,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:26:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:26:32,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:26:32,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:26:36,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:41,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:42,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:26:46,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:48,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:26:48,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:26:48,820 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:26:50,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:26:52,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:26:52,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:26:57,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:00,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:27:02,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:27:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:27:03,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:27:08,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:12,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:27:13,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:18,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:27:18,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:27:18,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:27:18,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:20,169 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:27:24,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:26,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:27:29,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:27:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:27:33,817 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:27:34,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:27:34,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:39,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:44,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:27:45,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:48,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:27:48,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:27:48,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:27:50,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:27:50,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:27:54,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:27:55,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:00,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:28:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:28:03,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:28:04,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:28:06,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:11,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:14,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:28:16,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:18,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:28:18,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:28:18,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:28:20,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:28:21,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:23,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:28:26,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:31,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:28:32,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:33,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:28:33,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:28:33,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:28:38,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:41,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:28:43,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:48,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:49,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:28:49,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:28:49,254 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:28:50,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:28:51,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:28:54,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:28:59,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:02,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:29:04,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:29:04,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:29:04,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:29:05,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:10,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:12,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:29:15,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:19,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:29:19,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:29:19,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:29:20,173 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:29:20,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:29:20,346 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:29:20,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:29:20,348 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:29:21,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:29:21,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:22,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:29:26,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:31,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:32,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:29:34,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:29:34,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:29:34,254 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:29:37,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:42,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:29:42,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:47,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:49,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:29:49,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:29:49,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:29:50,173 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:29:50,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:29:53,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:29:58,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:00,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:30:03,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:04,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:30:04,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:30:04,256 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:30:08,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:10,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:30:13,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:19,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:19,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:30:19,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:30:19,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:30:20,174 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:30:20,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:30:24,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:29,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:30,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:30:34,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:30:34,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:30:34,259 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:30:35,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:38,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:30:40,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:45,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:48,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:30:49,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:30:49,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:30:49,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:30:50,174 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:30:51,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:30:56,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:01,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:04,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:31:04,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:31:04,263 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:31:06,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:10,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:31:11,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:16,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:19,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:31:19,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:31:19,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:31:20,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:31:20,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:31:22,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:27,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:30,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:31:32,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:34,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:31:34,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:31:34,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:31:38,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:40,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:31:43,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:48,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:31:49,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:31:49,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:31:49,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:31:49,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:31:50,188 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:31:55,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:00,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:00,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:04,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:32:04,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:32:04,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:32:05,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:10,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:10,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:15,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:19,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:32:19,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:32:19,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:32:20,188 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:32:20,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:21,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:26,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:28,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:31,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:32:35,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:32:35,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:32:36,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:38,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:41,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:46,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:46,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:32:46,789 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:32:46,789 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:32:46,792 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:32:47,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:32:47,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:48,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:50,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:32:50,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:32:50,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:32:50,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:32:50,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:51,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:32:52,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:32:56,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:01,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:05,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:33:05,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:33:05,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:33:07,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:12,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:17,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:20,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:33:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:33:20,592 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:33:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:33:22,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:27,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:32,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:35,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:33:35,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:33:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:33:37,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:42,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:47,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:50,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:33:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:33:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:33:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:33:53,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:33:58,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:03,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:05,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:34:05,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:34:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:34:09,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:14,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:19,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:20,190 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:34:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:34:20,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:34:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:34:25,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:30,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:34:35,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:34:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:34:35,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:40,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:45,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:50,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:34:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:34:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:34:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:34:50,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:34:55,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:00,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:05,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:35:05,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:35:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:35:06,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:11,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:16,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:20,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:35:20,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:35:20,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:35:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:35:22,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:27,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:32,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:35:35,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:35:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:35:38,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:43,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:48,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:50,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:35:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:35:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:35:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:35:54,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:35:59,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:04,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:05,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:36:05,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:36:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:36:09,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:14,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:19,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:20,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:36:20,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:36:20,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:36:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:36:24,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:29,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:34,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:36:35,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:36:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:36:40,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:45,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:50,194 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:36:50,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:36:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:36:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:36:50,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:36:55,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:00,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:05,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:37:05,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:37:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:37:06,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:11,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:16,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:20,195 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:37:20,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:37:20,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:37:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:37:21,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:26,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:31,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:37:35,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:37:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:37:36,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:41,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:46,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:50,195 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:37:50,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:37:50,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:37:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:37:52,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:37:57,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:02,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:05,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:38:05,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:38:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:38:07,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:12,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:17,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:20,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:38:20,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:38:20,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:38:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:38:23,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:28,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:33,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:35,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:38:35,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:38:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:38:39,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:44,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:49,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:50,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:38:50,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:38:50,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:38:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:38:54,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:38:59,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:04,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:05,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:39:05,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:39:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:39:09,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:14,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:19,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:20,198 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:39:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:39:20,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:39:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:39:25,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:30,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:35,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:39:35,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:39:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:39:35,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:40,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:45,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:50,198 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:39:50,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:39:50,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:39:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:39:51,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:39:56,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:01,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:05,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:40:05,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:40:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:40:06,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:11,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:16,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:20,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:40:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:40:20,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:40:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:40:21,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:26,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:31,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:35,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:40:35,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:40:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:40:37,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:42,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:47,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:50,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:40:50,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:40:50,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:40:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:40:52,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:40:57,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:02,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:05,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:41:05,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:41:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:41:07,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:12,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:17,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:20,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:41:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:41:20,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:41:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:41:23,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:28,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:33,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:35,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:41:35,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:41:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:41:39,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:44,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:49,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:50,200 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:41:50,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:41:50,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:41:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:41:54,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:41:59,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:04,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:05,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:42:05,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:42:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:42:10,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:15,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:20,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:42:20,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:42:20,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:42:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:42:20,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:25,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:30,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:35,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:42:35,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:42:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:42:36,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:41,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:46,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:50,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:42:50,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:42:50,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:42:50,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:42:51,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:42:56,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:01,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:05,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:43:05,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:43:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:43:06,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:11,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:11,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:43:11,963 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:43:11,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:43:11,966 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:43:12,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:43:13,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:43:14,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:43:17,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:20,202 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:43:20,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:43:20,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:43:20,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:43:22,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:27,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:32,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:35,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:43:35,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:43:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:43:38,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:41,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:43:43,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:48,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:50,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:43:50,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:43:50,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:43:50,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:43:53,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:43:58,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:03,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:06,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:44:06,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:44:07,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:44:09,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:14,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:19,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:20,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:44:21,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:44:21,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:44:21,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:44:25,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:30,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:33,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:44:36,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:39,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:44:39,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:44:39,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:44:41,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:44:41,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:46,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:46,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:44:50,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:44:52,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:44:57,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:00,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:45:00,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:45:00,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:45:00,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:45:02,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:03,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:45:07,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:12,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:15,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:45:15,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:45:15,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:45:15,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:45:18,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:20,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:45:24,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:30,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:45:30,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:45:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:45:30,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:45:34,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:39,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:44,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:45,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:45:45,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:45:45,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:45:45,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:45:49,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:45:50,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:45:55,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:00,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:00,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:46:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:46:00,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:46:00,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:46:03,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:46:05,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:10,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:15,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:46:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:46:15,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:46:15,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:46:15,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:15,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:46:20,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:46:21,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:25,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:46:26,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:30,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:46:30,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:46:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:46:30,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:46:31,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:35,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:46:37,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:42,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:45,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:46:45,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:46:45,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:46:45,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:46:47,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:47,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:46:50,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:46:53,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:46:57,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:46:58,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:47:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:47:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:47:00,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:47:03,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:05,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:47:09,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:14,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:47:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:47:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:47:15,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:47:16,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:47:19,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:20,207 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:47:24,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:26,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:47:29,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:47:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:47:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:47:30,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:47:35,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:36,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:47:40,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:45,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:45,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:47:45,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:47:45,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:47:45,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:47:48,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:47:50,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:47:50,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:55,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:47:57,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:48:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:48:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:48:00,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:48:00,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:48:00,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:05,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:07,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:48:11,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:15,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:48:15,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:48:15,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:48:15,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:48:17,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:17,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:48:20,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:48:22,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:27,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:48:27,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:48:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:48:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:48:30,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:48:33,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:39,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:39,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:48:44,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:46,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:48:46,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:48:46,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:48:46,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:48:49,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:48:49,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:50,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:48:55,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:48:59,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:00,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:01,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:49:01,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:49:01,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:49:06,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:09,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:12,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:17,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:49:17,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:49:17,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:49:17,298 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:49:17,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:19,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:20,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:49:23,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:28,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:49:28,638 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:49:28,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:49:28,641 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:49:28,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:49:29,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:30,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:32,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:49:32,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:49:32,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:49:33,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:38,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:42,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:43,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:48,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:49:48,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:49:48,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:49:48,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:49:49,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:49:50,210 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:49:52,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:49:55,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:00,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:02,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:50:03,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:50:03,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:50:03,726 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:50:03,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:50:06,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:12,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:50:12,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:17,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:19,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:50:19,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:50:19,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:50:19,592 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:50:20,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:50:22,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:50:22,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:27,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:32,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:50:33,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:34,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:50:34,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:50:34,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:50:34,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:50:38,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:42,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:50:44,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:49,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:50,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:50:50,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:50:50,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:50:50,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:50:50,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:50:51,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:50:54,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:50:59,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:03,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:51:04,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:05,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:51:05,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:51:05,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:51:05,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:51:10,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:13,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:51:16,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:20,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:51:20,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:51:20,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:51:20,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:51:20,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:51:21,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:23,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:51:27,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:32,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:33,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:51:35,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:51:35,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:51:35,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:51:35,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:51:37,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:42,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:43,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:51:48,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:50,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:51:50,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:51:50,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:51:50,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:51:50,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:51:53,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:51:54,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:51:58,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:03,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:05,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:52:05,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:52:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:52:06,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:08,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:13,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:16,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:18,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:20,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:52:20,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:52:20,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:52:20,424 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:52:24,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:26,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:29,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:34,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:52:35,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:52:35,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:52:36,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:40,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:46,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:46,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:50,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:52:50,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:52:50,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:52:50,439 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:52:51,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:52:54,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:52:54,127 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:52:54,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:52:54,129 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:52:54,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:52:55,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:56,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:52:57,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:02,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:53:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:53:05,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:53:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:53:07,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:08,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:53:12,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:17,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:18,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:53:20,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:53:20,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:53:20,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:53:20,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:53:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:27,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:53:29,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:34,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:53:35,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:53:35,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:53:39,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:44,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:49,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:50,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:53:50,216 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:53:50,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:53:50,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:53:50,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:53:54,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:53:59,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:04,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:54:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:54:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:54:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:54:05,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:54:05,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:10,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:14,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:54:16,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:20,218 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:54:21,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:54:21,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:54:21,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:54:21,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:54:21,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:24,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:54:26,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:31,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:36,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:54:36,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:54:36,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:54:36,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:54:36,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:36,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:54:41,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:44,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:54:47,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:50,219 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:54:51,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:54:51,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:54:51,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:54:51,787 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:54:52,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:54:54,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:54:58,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:03,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:04,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:55:06,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:55:06,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:55:06,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:55:06,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:55:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:14,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:55:14,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:19,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:20,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:55:21,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:55:21,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:55:21,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:55:21,789 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:55:23,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:55:24,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:29,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:33,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:55:34,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:36,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:55:36,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:55:36,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:55:36,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:55:40,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:45,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:46,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:55:50,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:55:51,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:55:52,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:55:52,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:55:52,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:55:52,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:55:56,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:55:56,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:01,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:07,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:56:07,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:56:07,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:56:07,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:56:11,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:12,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:56:16,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:20,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:56:22,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:22,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:56:22,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:56:22,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:56:22,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:56:27,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:32,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:56:33,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:38,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:56:38,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:56:38,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:56:38,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:56:39,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:40,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:56:44,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:49,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:50,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 18:56:50,011 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-19 18:56:50,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-19 18:56:50,014 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 18:56:50,222 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-19 18:56:50,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-19 18:56:51,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:56:52,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:56:53,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:56:53,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:56:53,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:56:53,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:56:54,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:56:59,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:57:02,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:57:04,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:57:08,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 18:57:08,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 18:57:08,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 18:57:08,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status +2023-11-19 18:57:10,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 18:57:12,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-19 18:57:15,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report