diff --git "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" @@ -0,0 +1,3935 @@ +2021-07-16 00:59:47,118 INFO MainThread:761704 [internal.py:wandb_internal():88] W&B internal server running at pid: 761704, started at: 2021-07-16 00:59:47.118588 +2021-07-16 00:59:47,121 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: check_version +2021-07-16 00:59:47,121 INFO WriterThread:761704 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb +2021-07-16 00:59:47,122 DEBUG SenderThread:761704 [sender.py:send():179] send: header +2021-07-16 00:59:47,122 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: check_version +2021-07-16 00:59:47,162 DEBUG SenderThread:761704 [sender.py:send():179] send: run +2021-07-16 00:59:47,330 INFO SenderThread:761704 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files +2021-07-16 00:59:47,330 INFO SenderThread:761704 [sender.py:_start_run_threads():716] run started: 3bte92ft with start time 1626397186 +2021-07-16 00:59:47,330 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 00:59:47,330 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:59:47,331 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: run_start +2021-07-16 00:59:48,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 00:59:48,465 DEBUG HandlerThread:761704 [meta.py:__init__():39] meta init +2021-07-16 00:59:48,465 DEBUG HandlerThread:761704 [meta.py:__init__():53] meta init done +2021-07-16 00:59:48,465 DEBUG HandlerThread:761704 [meta.py:probe():210] probe +2021-07-16 00:59:48,466 DEBUG HandlerThread:761704 [meta.py:_setup_git():200] setup git +2021-07-16 00:59:48,501 DEBUG HandlerThread:761704 [meta.py:_setup_git():207] setup git done +2021-07-16 00:59:48,501 DEBUG HandlerThread:761704 [meta.py:_save_pip():57] save pip +2021-07-16 00:59:48,501 DEBUG HandlerThread:761704 [meta.py:_save_pip():71] save pip done +2021-07-16 00:59:48,502 DEBUG HandlerThread:761704 [meta.py:probe():252] probe done +2021-07-16 00:59:48,505 DEBUG SenderThread:761704 [sender.py:send():179] send: files +2021-07-16 00:59:48,505 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-16 00:59:48,511 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:59:48,512 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:59:48,640 DEBUG SenderThread:761704 [sender.py:send():179] send: config +2021-07-16 00:59:48,641 DEBUG SenderThread:761704 [sender.py:send():179] send: config +2021-07-16 00:59:48,641 DEBUG SenderThread:761704 [sender.py:send():179] send: config +2021-07-16 00:59:48,934 INFO Thread-11 :761704 [upload_job.py:push():137] Uploaded file /tmp/tmpvg0t2zt4wandb/3fzrz0vd-wandb-metadata.json +2021-07-16 00:59:49,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-metadata.json +2021-07-16 00:59:49,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 00:59:49,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/requirements.txt +2021-07-16 01:00:03,739 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:00:03,739 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:00:04,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:00:06,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:00:16,553 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:00:18,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/config.yaml +2021-07-16 01:00:18,894 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:00:18,894 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:00:34,025 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:00:34,025 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:00:46,631 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:00:49,170 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:00:49,171 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:01:04,306 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:01:04,306 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:01:16,706 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:01:19,442 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:01:19,442 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:01:34,577 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:01:34,577 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:01:46,781 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:01:49,716 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:01:49,716 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:02:04,848 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:02:04,849 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:02:16,858 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:02:19,983 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:02:19,983 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:02:35,116 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:02:35,117 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:02:46,932 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:02:50,249 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:02:50,249 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:03:05,383 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:03:05,384 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:03:17,007 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:03:20,517 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:03:20,517 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:03:35,648 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:03:35,649 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:03:47,083 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:03:50,786 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:03:50,786 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:04:05,916 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:04:05,917 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:04:17,159 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:04:21,051 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:04:21,051 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:04:36,185 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:04:36,185 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:04:47,235 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:04:51,320 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:04:51,320 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:05:06,454 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:05:06,454 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:05:17,308 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:05:21,588 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:05:21,589 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:05:36,721 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:05:36,722 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:05:38,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:05:47,383 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:05:51,869 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:05:51,869 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:06:07,010 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:06:07,010 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:06:17,459 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:06:22,142 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:06:22,142 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:06:37,273 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:06:37,273 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:06:47,533 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:06:52,405 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:06:52,406 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:07:07,536 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:07:07,536 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:07:17,613 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:07:22,668 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:07:22,668 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:07:37,800 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:07:37,800 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:07:43,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:07:47,689 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:07:52,947 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:07:52,947 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:08:08,090 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:08:08,090 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:08:17,767 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:08:23,221 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:08:23,221 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:08:38,354 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:08:38,354 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:08:47,847 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:08:53,490 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:08:53,490 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:09:08,624 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:09:08,624 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:09:17,929 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:09:23,758 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:09:23,759 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:09:38,891 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:09:38,891 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:09:48,019 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:09:54,024 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:09:54,024 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:10:01,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:03,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:05,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:07,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:09,155 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:10:09,156 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:10:18,097 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:10:18,898 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:10:18,899 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:10:18,900 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:10:19,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:19,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:10:21,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:23,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:24,287 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:10:24,287 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:10:25,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:27,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:38,938 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:10:38,939 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:10:38,939 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:10:39,430 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:10:39,430 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:10:39,588 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:10:41,589 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:43,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:45,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:47,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:10:48,175 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:10:54,562 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:10:54,562 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:10:58,977 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:10:58,977 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:10:58,978 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:10:59,596 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:11:01,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:03,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:05,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:07,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:09,693 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:11:09,694 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:11:18,253 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:11:19,016 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:11:19,017 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:11:19,017 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:11:19,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:19,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:11:21,605 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:23,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:24,828 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:11:24,828 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:11:25,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:27,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:39,055 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:11:39,056 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:11:39,056 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:11:39,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:39,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:11:39,972 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:11:39,972 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:11:41,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:43,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:45,614 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:47,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:48,327 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:11:55,107 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:11:55,107 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:11:59,098 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:11:59,099 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:11:59,099 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:11:59,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:11:59,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:12:01,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:03,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:05,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:07,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:10,240 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:12:10,240 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:12:18,402 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:12:19,137 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:12:19,138 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:12:19,138 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:12:19,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:19,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:12:21,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:23,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:25,378 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:12:25,378 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:12:25,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:27,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:39,177 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:12:39,177 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:12:39,178 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:12:39,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:12:40,550 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:12:40,551 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:12:41,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:43,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:45,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:47,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:12:48,469 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:12:55,692 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:12:55,692 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:12:59,215 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:12:59,216 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:12:59,216 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:12:59,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:13:01,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:03,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:05,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:07,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:10,823 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:13:10,824 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:13:18,539 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:13:19,251 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:13:19,251 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:13:19,252 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:13:19,653 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:13:21,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:23,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:25,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:25,954 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:13:25,954 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:13:27,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:39,288 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:13:39,288 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:13:39,289 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:13:39,661 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:13:41,111 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:13:41,111 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:13:42,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:44,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:46,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:48,612 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:13:48,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:13:56,251 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:13:56,251 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:13:59,325 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:13:59,325 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:13:59,326 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:13:59,669 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:14:02,670 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:04,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:06,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:08,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:11,382 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:14:11,383 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:14:18,683 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:14:19,371 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:14:19,371 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:14:19,372 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:14:19,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:14:20,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:22,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:24,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:26,515 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:14:26,516 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:14:26,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:28,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:39,411 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:14:39,411 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:14:39,412 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:14:39,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:14:40,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:41,687 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:14:41,687 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:14:42,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:44,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:46,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:48,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:14:48,754 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:14:56,819 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:14:56,819 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:14:59,451 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:14:59,451 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:14:59,451 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:14:59,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:15:00,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:02,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:04,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:06,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:08,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:11,951 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:15:11,951 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:15:18,828 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:15:19,492 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:15:19,492 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:15:19,492 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:15:19,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:15:20,709 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:22,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:24,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:26,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:27,082 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:15:27,082 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:15:28,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:39,530 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:15:39,531 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:15:39,531 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:15:39,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:15:40,718 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:42,217 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:15:42,217 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:15:42,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:44,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:46,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:48,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:15:48,904 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:15:57,349 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:15:57,350 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:15:59,574 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:15:59,574 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:15:59,575 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:15:59,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:16:00,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:02,727 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:04,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:06,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:08,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:12,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:16:12,488 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:16:18,979 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:16:19,610 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:16:19,610 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:16:19,611 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:16:19,734 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:16:20,734 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:22,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:24,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:26,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:27,620 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:16:27,621 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:16:28,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:39,646 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:16:39,646 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:16:39,647 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:16:39,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:16:40,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:42,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:42,757 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:16:42,758 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:16:44,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:46,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:48,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:16:49,057 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:16:57,892 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:16:57,893 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:16:59,685 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:16:59,685 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:16:59,686 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:16:59,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:17:00,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:02,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:04,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:06,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:08,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:13,024 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:17:13,024 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:17:19,135 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:17:19,725 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:17:19,725 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:17:19,726 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:17:19,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:17:22,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:24,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:26,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:28,158 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:17:28,159 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:17:28,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:39,762 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:17:39,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:17:39,763 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:17:39,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:17:40,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:42,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:43,297 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:17:43,298 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:17:44,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:46,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:48,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:17:49,213 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:17:58,462 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:17:58,463 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:17:59,802 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:17:59,802 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:17:59,803 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:18:00,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:00,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:18:02,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:04,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:06,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:08,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:13,593 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:18:13,593 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:18:19,290 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:18:19,839 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:18:19,839 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:18:19,840 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:18:20,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:20,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:18:22,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:24,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:26,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:28,724 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:18:28,724 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:18:28,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:39,876 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:18:39,877 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:18:39,877 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:18:40,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:40,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:18:42,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:43,855 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:18:43,856 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:18:44,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:46,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:48,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:18:49,366 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:18:58,987 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:18:58,988 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:18:59,914 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:18:59,914 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:18:59,915 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:19:00,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:00,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:19:02,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:04,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:06,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:08,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:14,120 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:19:14,120 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:19:19,441 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:19:19,950 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:19:19,950 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:19:19,951 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:19:20,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:20,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:19:22,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:24,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:26,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:28,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:29,250 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:19:29,250 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:19:39,987 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:19:39,987 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:19:39,988 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:19:40,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:40,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:19:42,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:44,398 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:19:44,398 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:19:44,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:46,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:48,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:19:49,517 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:19:59,532 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:19:59,532 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:20:00,031 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:20:00,031 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:20:00,031 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:20:00,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:00,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:20:02,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:04,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:06,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:08,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:14,663 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:20:14,663 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:20:19,593 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:20:20,075 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:20:20,075 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:20:20,075 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:20:20,839 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:20,839 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:20:22,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:24,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:26,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:28,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:29,794 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:20:29,795 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:20:40,116 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:20:40,116 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:20:40,116 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:20:40,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:40,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:20:42,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:44,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:44,929 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:20:44,929 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:20:46,849 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:48,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:20:49,670 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:21:00,060 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:21:00,060 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:21:00,196 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:21:00,196 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:21:00,197 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:21:00,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:00,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:21:02,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:04,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:06,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:08,857 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:15,212 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:21:15,212 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:21:19,748 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:21:20,193 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:21:20,194 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:21:20,194 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:21:20,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:20,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:21:22,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:24,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:26,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:28,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:30,343 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:21:30,344 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:21:40,231 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:21:40,231 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:21:40,232 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:21:40,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:40,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:21:43,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:45,474 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:21:45,474 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:21:45,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:47,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:21:49,825 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:21:49,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:00,272 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:22:00,273 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:22:00,273 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:22:00,639 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:22:00,639 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:22:00,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:00,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:22:01,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:03,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:05,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:07,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:09,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:15,781 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:22:15,781 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:22:19,902 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:22:20,309 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:22:20,310 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:22:20,310 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:22:20,883 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:20,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:22:21,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:23,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:25,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:27,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:29,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:30,919 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:22:30,919 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:22:40,346 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:22:40,347 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:22:40,347 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:22:40,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:22:41,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:43,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:45,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:46,054 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:22:46,055 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:22:47,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:49,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:22:49,979 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:23:00,384 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:23:00,384 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:23:00,385 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:23:00,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:23:01,196 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:23:01,196 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:23:01,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:03,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:05,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:07,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:09,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:16,330 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:23:16,331 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:23:20,057 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:23:20,422 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:23:20,422 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:23:20,422 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:23:20,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:23:21,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:23,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:25,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:27,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:29,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:31,464 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:23:31,464 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:23:40,459 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:23:40,459 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:23:40,460 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:23:40,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:23:41,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:43,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:45,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:46,596 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:23:46,596 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:23:47,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:49,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:23:50,132 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:24:00,500 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:24:00,501 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:24:00,501 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:24:00,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:24:01,727 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:24:01,727 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:24:01,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:03,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:05,926 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:07,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:09,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:16,893 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:24:16,893 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:24:20,209 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:24:20,538 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:24:20,538 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:24:20,539 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:24:20,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:24:21,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:23,934 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:25,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:27,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:29,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:32,025 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:24:32,025 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:24:40,588 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:24:40,588 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:24:40,589 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:24:40,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:24:41,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:43,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:45,943 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:47,156 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:24:47,156 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:24:47,943 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:49,944 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:24:50,287 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:25:00,626 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:25:00,626 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:25:00,626 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:25:00,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:25:01,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:02,306 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:25:02,306 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:25:03,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:05,951 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:07,952 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:09,953 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:17,452 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:25:17,452 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:25:20,366 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:25:20,667 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:25:20,667 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:25:20,668 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:25:20,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:25:21,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:23,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:25,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:27,960 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:29,961 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:32,594 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:25:32,594 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:25:40,712 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:25:40,712 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:25:40,713 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:25:40,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:25:41,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:43,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:45,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:47,725 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:25:47,726 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:25:47,969 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:49,970 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:25:50,437 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:26:00,753 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:26:00,754 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:26:00,754 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:26:00,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:26:01,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:02,859 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:26:02,859 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:26:03,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:05,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:07,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:09,979 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:18,031 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:26:18,031 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:26:20,507 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:26:20,796 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:26:20,796 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:26:20,797 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:26:20,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:26:21,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:23,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:25,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:27,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:29,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:33,166 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:26:33,166 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:26:40,836 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:26:40,836 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:26:40,836 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:26:40,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:26:41,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:43,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:45,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:47,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:48,296 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:26:48,297 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:26:49,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:26:50,577 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:27:00,882 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:27:00,883 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:27:00,883 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:27:01,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:27:02,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:03,441 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:27:03,442 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:27:04,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:06,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:08,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:10,003 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:18,578 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:27:18,578 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:27:20,645 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:27:20,929 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:27:20,929 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:27:20,930 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:27:21,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:27:22,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:24,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:26,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:28,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:30,012 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:33,715 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:27:33,716 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:27:40,966 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:27:40,966 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:27:40,967 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:27:41,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:27:42,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:44,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:46,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:48,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:48,850 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:27:48,850 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:27:50,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:27:50,715 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:28:01,035 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:28:01,035 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:28:01,036 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:28:02,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:02,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:28:03,990 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:28:03,990 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:28:04,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:06,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:08,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:10,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:19,120 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:28:19,121 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:28:20,784 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:28:21,119 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:28:21,119 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:28:21,119 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:28:22,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:22,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:28:24,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:26,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:28,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:30,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:34,251 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:28:34,252 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:28:41,161 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:28:41,161 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:28:41,162 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:28:42,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:42,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:28:44,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:46,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:48,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:49,382 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:28:49,382 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:28:50,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:28:50,857 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:29:01,214 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:29:01,215 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:29:01,215 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:29:02,050 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:02,050 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:29:04,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:04,516 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:29:04,517 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:29:06,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:08,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:10,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:19,647 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:29:19,648 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:29:20,936 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:29:21,252 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:29:21,253 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:29:21,253 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:29:22,058 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:22,058 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:29:24,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:26,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:28,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:30,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:34,781 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:29:34,782 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:29:41,290 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:29:41,290 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:29:41,291 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:29:42,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:42,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:29:44,067 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:46,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:48,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:49,914 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:29:49,914 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:29:50,069 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:29:51,015 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:30:01,328 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:30:01,328 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:30:01,329 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:30:02,074 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:02,075 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:30:04,075 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:05,049 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:30:05,049 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:30:06,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:08,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:10,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:20,183 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:30:20,183 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:30:21,096 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:30:21,366 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:30:21,367 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:30:21,367 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:30:22,082 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:22,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:30:24,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:26,084 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:28,085 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:30,086 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:35,314 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:30:35,315 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:30:41,401 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:30:41,402 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:30:41,402 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:30:42,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:30:43,091 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:45,092 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:47,092 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:49,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:50,447 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:30:50,447 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:30:51,094 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:30:51,174 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:31:01,439 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:31:01,439 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:31:01,440 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:31:02,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:31:03,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:05,100 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:05,581 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:31:05,582 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:31:07,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:09,102 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:11,102 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:20,714 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:31:20,715 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:31:21,252 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:31:21,478 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:31:21,479 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:31:21,479 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:31:22,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:31:23,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:25,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:27,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:29,110 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:31,110 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:35,846 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:31:35,846 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:31:41,516 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:31:41,516 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:31:41,517 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:31:42,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:31:43,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:45,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:47,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:49,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:50,978 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:31:50,978 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:31:51,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:31:51,329 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:32:01,556 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:32:01,557 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:32:01,557 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:32:02,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:32:03,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:05,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:06,114 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:32:06,114 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:32:07,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:09,126 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:11,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:21,245 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:32:21,246 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:32:21,406 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:32:21,600 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:32:21,601 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:32:21,601 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:32:22,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:32:23,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:25,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:27,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:29,134 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:31,135 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:36,378 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:32:36,378 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:32:41,638 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:32:41,639 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:32:41,639 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:32:42,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:32:43,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:45,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:47,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:49,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:51,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:32:51,484 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:32:51,515 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:32:51,516 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:33:01,683 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:33:01,684 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:33:01,684 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:33:02,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:33:03,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:05,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:06,647 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:33:06,648 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:33:07,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:09,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:11,152 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:21,562 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:33:21,724 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:33:21,724 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:33:21,725 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:33:21,783 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:33:21,784 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:33:22,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:33:23,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:25,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:27,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:29,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:31,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:36,916 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:33:36,916 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:33:41,761 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:33:41,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:33:41,762 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:33:42,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:33:43,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:45,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:47,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:49,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:51,169 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:33:51,640 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:33:52,048 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:33:52,049 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:34:01,800 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:34:01,800 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:34:01,801 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:34:02,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:34:03,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:05,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:07,176 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:07,178 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:34:07,178 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:34:09,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:11,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:21,718 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:34:21,838 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:34:21,839 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:34:21,839 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:34:22,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:34:22,327 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:34:22,328 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:34:23,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:25,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:27,185 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:29,186 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:31,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:37,459 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:34:37,459 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:34:41,884 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:34:41,884 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:34:41,885 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:34:42,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:34:43,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:45,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:47,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:49,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:51,195 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:34:51,796 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:34:52,633 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:34:52,633 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:35:01,921 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:35:01,921 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:35:01,922 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:35:02,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:35:03,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:05,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:07,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:07,766 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:35:07,766 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:35:09,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:11,204 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:21,874 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:35:21,958 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:35:21,958 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:35:21,959 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:35:22,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:35:22,943 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:35:22,944 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:35:23,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:25,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:27,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:29,212 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:31,213 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:38,084 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:35:38,084 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:35:41,997 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:35:41,998 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:35:41,998 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:35:42,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:35:43,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:45,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:47,220 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:49,220 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:51,221 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:35:51,951 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:35:53,215 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:35:53,216 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:36:02,041 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:36:02,041 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:36:02,041 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:36:02,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:36:03,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:05,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:07,228 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:08,346 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:36:08,347 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:36:09,229 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:11,230 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:22,029 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:36:22,081 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:36:22,081 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:36:22,082 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:36:22,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:36:23,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:23,483 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:36:23,483 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:36:25,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:27,237 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:29,238 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:31,239 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:38,617 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:36:38,617 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:36:42,121 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:36:42,121 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:36:42,122 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:36:42,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:36:43,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:45,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:47,246 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:49,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:51,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:36:52,107 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:36:53,751 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:36:53,751 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:37:02,164 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:37:02,164 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:37:02,165 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:37:02,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:37:03,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:05,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:07,254 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:08,881 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:37:08,882 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:37:09,255 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:11,256 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:22,185 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:37:22,200 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:37:22,200 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:37:22,201 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:37:22,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:37:23,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:24,013 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:37:24,013 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:37:25,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:27,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:29,263 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:31,264 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:39,155 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:37:39,155 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:37:42,246 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:37:42,247 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:37:42,247 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:37:42,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:37:43,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:45,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:47,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:49,271 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:51,272 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:37:52,262 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:37:54,289 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:37:54,289 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:38:02,289 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:38:02,290 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:38:02,290 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:38:03,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:03,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:38:05,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:07,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:09,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:09,421 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:38:09,421 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:38:11,281 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:22,341 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:38:22,369 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:38:22,369 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:38:22,370 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:38:23,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:23,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:38:24,555 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:38:24,555 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:38:25,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:27,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:29,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:31,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:39,695 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:38:39,696 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:38:42,408 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:38:42,408 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:38:42,408 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:38:43,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:43,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:38:45,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:47,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:49,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:51,298 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:38:52,417 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:38:54,827 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:38:54,827 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:39:02,446 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:39:02,446 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:39:02,446 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:39:03,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:03,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:39:05,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:07,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:09,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:09,960 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:39:09,961 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:39:11,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:22,485 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:39:22,485 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:39:22,487 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:39:22,495 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:39:23,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:23,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:39:25,098 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:39:25,098 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:39:25,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:27,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:29,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:31,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:40,229 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:39:40,230 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:39:42,533 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:39:42,533 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:39:42,534 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:39:43,334 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:43,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:39:45,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:47,336 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:49,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:51,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:39:52,572 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:39:55,364 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:39:55,364 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:40:02,571 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:40:02,571 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:40:02,572 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:40:03,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:03,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:40:05,343 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:07,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:09,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:10,501 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:40:10,502 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:40:11,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:22,610 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:40:22,610 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:40:22,613 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:40:22,647 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:40:23,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:23,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:40:25,351 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:25,641 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:40:25,641 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:40:27,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:29,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:31,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:40,773 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:40:40,773 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:40:42,651 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:40:42,651 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:40:42,652 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:40:43,358 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:43,358 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:40:45,359 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:47,359 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:49,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:51,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:40:52,719 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:40:55,903 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:40:55,903 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:41:02,693 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:41:02,693 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:41:02,694 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:41:03,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:41:04,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:06,367 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:08,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:10,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:11,036 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:41:11,036 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:41:12,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:22,729 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:41:22,729 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:41:22,732 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:41:22,791 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:41:23,374 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:41:24,374 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:26,169 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:41:26,169 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:41:26,375 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:28,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:30,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:32,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:41,300 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:41:41,301 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:41:42,774 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:41:42,774 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:41:42,775 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:41:43,382 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:41:44,382 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:46,383 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:48,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:50,385 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:52,385 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:41:52,869 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:41:56,435 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:41:56,435 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:42:02,810 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:42:02,810 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:42:02,811 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:42:03,390 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:42:04,390 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:06,391 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:08,392 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:10,393 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:11,568 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:42:11,569 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:42:12,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:22,847 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:42:22,847 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:42:22,848 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:42:22,947 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:42:23,398 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:42:24,399 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:26,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:26,707 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:42:26,707 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:42:28,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:30,401 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:32,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:41,840 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:42:41,840 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:42:42,884 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:42:42,884 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:42:42,885 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:42:43,407 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:42:44,407 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:46,408 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:48,409 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:50,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:52,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:42:53,025 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:42:56,973 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:42:56,973 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:43:02,924 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:43:02,924 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:43:02,924 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:43:03,415 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:43:04,415 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:06,416 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:08,417 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:10,418 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:12,106 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:43:12,106 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:43:12,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:22,964 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:43:22,964 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:43:22,964 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:43:23,102 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:43:23,423 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:43:24,424 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:26,424 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:27,237 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:43:27,237 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:43:28,425 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:30,426 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:32,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:42,368 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:43:42,369 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:43:43,002 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:43:43,002 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:43:43,003 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:43:43,431 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:43:44,432 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:46,433 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:48,434 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:50,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:52,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:43:53,180 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:43:57,500 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:43:57,501 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:44:03,043 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:44:03,044 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:44:03,044 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:44:03,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:44:04,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:06,441 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:08,442 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:10,443 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:12,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:12,633 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:44:12,634 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:44:23,088 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:44:23,088 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:44:23,089 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:44:23,259 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:44:23,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:44:24,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:26,449 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:27,765 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:44:27,766 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:44:28,450 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:30,451 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:32,452 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:42,897 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:44:42,897 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:44:43,127 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:44:43,127 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:44:43,127 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:44:43,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:44:44,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:46,457 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:48,458 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:50,459 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:52,460 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:44:53,337 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:44:58,028 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:44:58,029 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:45:03,166 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:45:03,166 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:45:03,167 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:45:03,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:45:04,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:06,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:08,466 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:10,467 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:12,468 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:13,160 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:45:13,160 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:45:23,205 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:45:23,206 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:45:23,206 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:45:23,415 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:45:23,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:45:24,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:26,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:28,295 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:45:28,295 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:45:28,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:30,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:32,476 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:43,250 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:45:43,250 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:45:43,251 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:45:43,430 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:45:43,430 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:45:43,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:45:44,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:46,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:48,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:50,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:52,484 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:45:53,490 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:45:58,575 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:45:58,575 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:46:03,287 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:46:03,287 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:46:03,288 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:46:03,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:46:04,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:06,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:08,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:10,492 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:12,493 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:13,712 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:46:13,712 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:46:23,325 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:46:23,326 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:46:23,326 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:46:23,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:46:23,568 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:46:24,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:26,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:28,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:28,843 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:46:28,843 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:46:30,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:32,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:43,370 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:46:43,371 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:46:43,371 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:46:43,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:46:43,994 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:46:43,995 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:46:44,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:46,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:48,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:50,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:52,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:46:53,647 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:46:59,127 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:46:59,128 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:47:03,406 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:47:03,406 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:47:03,407 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:47:03,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:47:04,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:06,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:08,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:10,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:12,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:14,266 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:47:14,266 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:47:23,443 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:47:23,444 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:47:23,444 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:47:23,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:47:23,724 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:47:24,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:26,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:28,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:29,397 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:47:29,397 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:47:30,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:32,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:43,490 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:47:43,490 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:47:43,490 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:47:43,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:47:44,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:44,562 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:47:44,562 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:47:46,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:48,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:50,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:52,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:47:53,802 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:47:59,700 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:47:59,701 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:48:03,533 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:48:03,533 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:48:03,534 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:48:03,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:48:04,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:06,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:08,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:10,542 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:12,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:14,832 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:48:14,832 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:48:23,569 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:48:23,569 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:48:23,570 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:48:23,879 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:48:24,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:24,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:48:26,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:28,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:29,981 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:48:29,982 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:48:30,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:32,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:43,606 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:48:43,607 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:48:43,607 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:48:44,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:44,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:48:45,125 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:48:45,126 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:48:46,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:48,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:50,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:52,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:48:53,956 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:49:00,290 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:49:00,290 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:49:03,645 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:49:03,645 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:49:03,645 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:49:04,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:04,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:49:06,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:08,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:10,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:12,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:15,423 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:49:15,424 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:49:23,683 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:49:23,684 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:49:23,684 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:49:24,032 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:49:24,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:24,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:49:26,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:28,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:30,570 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:49:30,570 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:49:30,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:32,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:43,721 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:49:43,722 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:49:43,722 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:49:44,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:44,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:49:45,734 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:49:45,735 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:49:46,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:48,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:50,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:52,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:49:54,102 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:50:01,058 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:50:01,059 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:50:03,760 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:50:03,760 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:50:03,761 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:50:04,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:04,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:50:16,191 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:50:16,191 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:50:24,332 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:50:24,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:26,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:28,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:30,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:31,546 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:50:31,547 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:50:33,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:35,603 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:37,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:39,605 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:41,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:43,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:45,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:46,680 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:50:46,680 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:50:47,608 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:49,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:51,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:53,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:54,409 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:50:55,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:57,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:50:59,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:01,614 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:01,848 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:51:01,848 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:51:03,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:05,616 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:07,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:09,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:11,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:13,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:15,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:16,982 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:51:16,983 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:51:17,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:19,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:21,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:23,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:24,484 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:51:26,175 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:51:26,176 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:51:26,176 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:51:26,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:51:27,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:29,626 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:31,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:32,115 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:51:32,115 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:51:33,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:35,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:46,225 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:51:46,226 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:51:46,226 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:51:46,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:51:47,293 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:51:47,293 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:51:47,634 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:49,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:51,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:53,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:51:54,562 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:51:55,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:02,428 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:52:02,428 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:52:06,275 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:52:06,275 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:52:06,277 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:52:06,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:52:07,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:09,644 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:11,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:13,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:15,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:17,561 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:52:17,561 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:52:24,638 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:52:26,316 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:52:26,316 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:52:26,317 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:52:26,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:52:27,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:29,652 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:31,653 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:32,691 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:52:32,691 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:52:33,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:35,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:46,362 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:52:46,363 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:52:46,363 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:52:46,659 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:52:47,660 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:47,866 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:52:47,866 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:52:49,661 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:51,661 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:53,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:52:54,713 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:53:03,005 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:53:03,005 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:53:06,407 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:53:06,407 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:53:06,407 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:53:06,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:53:07,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:09,669 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:11,670 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:13,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:18,138 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:53:18,138 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:53:24,791 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:53:26,444 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:53:26,445 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:53:26,445 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:53:26,676 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:53:27,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:29,678 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:31,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:33,272 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:53:33,273 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:53:33,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:46,483 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:53:46,483 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:53:46,484 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:53:46,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:53:47,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:48,406 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:53:48,406 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:53:49,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:51,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:53,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:53:54,868 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:53:55,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:03,545 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:54:03,546 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:54:06,529 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:54:06,529 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:54:06,530 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:54:06,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:54:07,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:09,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:11,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:13,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:15,698 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:18,683 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:54:18,683 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:54:24,946 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:54:26,569 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:54:26,569 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:54:26,569 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:54:26,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:54:27,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:28,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:30,704 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:32,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:33,816 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:54:33,817 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:54:34,706 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:36,707 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:46,605 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:54:46,606 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:54:46,606 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:54:46,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:54:47,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:48,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:48,977 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:54:48,978 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:54:50,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:52,714 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:54,715 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:54:55,026 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:54:56,716 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:04,123 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:55:04,124 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:55:06,643 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:55:06,643 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:55:06,644 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:55:06,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:55:07,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:08,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:10,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:12,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:14,724 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:19,258 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:55:19,258 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:55:25,104 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:55:26,680 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:55:26,680 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:55:26,680 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:55:26,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:55:27,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:28,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:30,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:32,732 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:34,392 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:55:34,392 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:55:34,733 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:46,719 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:55:46,719 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:55:46,719 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:55:46,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:55:47,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:48,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:49,523 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:55:49,524 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:55:50,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:52,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:54,741 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:55:55,181 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:56:04,657 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:56:04,657 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:56:06,757 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:56:06,758 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:56:06,758 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:56:07,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:07,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:56:08,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:10,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:12,749 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:14,750 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:16,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:19,787 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:56:19,787 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:56:25,259 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:56:26,794 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:56:26,795 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:56:26,795 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:56:27,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:27,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:56:28,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:30,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:32,758 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:34,759 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:34,965 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:56:34,966 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:56:36,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:46,830 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:56:46,831 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:56:46,831 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:56:47,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:47,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:56:48,765 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:50,096 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:56:50,096 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:56:50,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:52,767 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:54,768 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:56:55,336 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:56:56,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:05,228 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:57:05,229 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:57:06,869 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:57:06,869 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:57:06,869 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:57:07,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:07,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:57:08,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:10,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:12,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:14,777 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:16,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:20,528 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:57:20,528 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:57:25,415 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:57:26,906 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:57:26,906 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:57:26,907 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:57:27,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:27,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:57:28,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:30,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:32,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:34,786 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:35,659 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:57:35,660 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:57:36,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:46,946 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:57:46,947 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:57:46,947 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:57:47,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:47,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:57:48,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:50,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:50,794 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:57:50,794 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:57:52,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:54,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:57:55,492 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:58:05,931 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:58:05,931 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:58:06,988 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:58:06,989 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:58:06,989 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:58:07,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:07,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:58:08,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:10,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:12,803 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:14,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:21,065 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:58:21,066 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:58:25,570 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:58:27,034 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:58:27,034 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:58:27,034 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:58:27,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:58:28,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:30,811 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:32,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:34,813 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:36,214 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:58:36,215 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:58:36,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:47,079 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:58:47,079 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:58:47,080 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:58:47,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:58:48,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:50,820 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:51,591 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:58:51,591 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:58:52,821 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:54,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:58:55,646 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:58:56,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:06,727 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:59:06,727 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:59:07,117 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:59:07,118 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:59:07,118 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:59:07,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:59:08,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:10,829 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:12,830 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:14,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:21,860 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:59:21,860 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:59:25,725 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 01:59:27,156 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:59:27,157 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:59:27,157 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:59:27,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:59:28,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:30,838 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:32,839 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:34,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:37,003 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:59:37,003 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:59:47,195 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 01:59:47,196 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 01:59:47,196 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 01:59:47,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 01:59:48,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:50,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:52,136 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 01:59:52,136 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 01:59:52,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:54,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 01:59:55,802 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:00:07,232 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:00:07,233 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:00:07,234 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:00:07,274 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:00:07,275 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:00:07,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:00:08,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:10,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:12,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:14,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:22,408 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:00:22,408 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:00:25,879 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:00:27,272 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:00:27,273 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:00:27,273 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:00:27,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:00:28,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:30,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:32,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:34,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:37,561 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:00:37,561 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:00:47,318 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:00:47,318 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:00:47,319 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:00:47,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:00:48,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:50,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:52,696 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:00:52,697 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:00:52,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:54,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:00:55,955 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:01:07,356 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:01:07,357 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:01:07,357 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:01:07,847 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:01:07,847 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:01:07,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:01:08,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:09,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:11,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:13,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:15,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:22,980 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:01:22,980 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:01:26,028 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:01:27,401 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:01:27,402 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:01:27,402 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:01:27,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:01:28,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:29,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:31,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:33,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:35,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:38,112 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:01:38,112 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:01:47,440 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:01:47,441 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:01:47,441 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:01:47,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:01:48,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:49,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:51,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:53,245 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:01:53,246 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:01:53,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:55,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:01:56,102 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:02:07,479 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:02:07,479 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:02:07,480 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:02:07,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:02:08,414 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:02:08,414 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:02:08,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:09,905 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:11,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:13,907 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:15,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:23,554 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:02:23,554 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:02:26,178 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:02:27,519 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:02:27,519 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:02:27,519 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:02:27,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:02:28,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:29,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:31,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:33,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:35,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:38,689 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:02:38,690 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:02:47,559 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:02:47,560 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:02:47,561 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:02:47,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:02:48,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:49,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:51,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:53,829 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:02:53,829 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:02:53,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:55,926 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:02:56,253 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:03:07,598 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:03:07,598 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:03:07,599 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:03:07,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:03:08,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:08,980 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:03:08,981 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:03:09,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:11,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:13,934 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:15,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:24,117 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:03:24,117 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:03:26,328 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:03:27,636 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:03:27,636 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:03:27,637 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:03:27,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:03:28,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:29,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:31,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:33,943 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:35,944 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:39,251 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:03:39,251 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:03:47,672 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:03:47,673 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:03:47,673 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:03:47,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:03:48,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:49,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:51,951 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:53,952 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:54,381 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:03:54,381 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:03:55,953 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:03:56,404 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:04:07,709 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:04:07,710 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:04:07,710 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:04:07,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:04:08,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:09,555 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:04:09,556 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:04:09,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:11,960 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:13,961 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:15,962 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:24,687 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:04:24,688 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:04:26,482 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:04:27,747 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:04:27,748 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:04:27,748 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:04:27,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:04:28,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:29,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:31,969 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:33,970 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:35,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:39,817 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:04:39,818 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:04:47,798 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:04:47,798 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:04:47,799 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:04:47,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:04:48,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:49,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:51,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:53,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:54,949 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:04:54,949 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:04:55,979 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:04:56,559 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:05:07,888 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:05:07,888 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:05:07,889 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:05:07,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:05:08,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:09,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:10,092 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:05:10,092 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:05:11,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:13,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:15,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:25,225 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:05:25,225 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:05:26,637 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:05:27,926 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:05:27,927 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:05:27,927 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:05:27,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:05:28,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:29,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:31,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:33,996 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:35,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:40,357 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:05:40,358 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:05:47,962 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:05:47,963 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:05:47,963 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:05:48,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:05:49,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:50,003 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:52,003 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:54,004 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:55,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:05:55,488 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:05:56,005 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:05:56,714 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:06:08,004 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:06:08,004 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:06:08,005 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:06:08,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:06:09,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:10,012 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:10,618 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:06:10,618 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:06:12,013 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:14,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:16,015 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:25,747 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:06:25,748 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:06:26,792 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:06:28,043 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:06:28,044 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:06:28,044 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:06:29,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:29,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:06:30,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:32,021 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:34,022 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:36,023 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:40,879 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:06:40,879 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:06:48,082 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:06:48,083 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:06:48,083 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:06:49,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:49,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:06:50,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:52,030 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:54,031 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:56,013 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:06:56,013 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:06:56,032 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:06:56,870 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:07:08,120 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:07:08,120 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:07:08,121 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:07:09,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:09,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:07:10,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:11,141 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:07:11,141 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:07:12,039 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:14,040 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:16,040 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:26,272 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:07:26,272 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:07:26,947 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:07:28,164 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:07:28,164 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:07:28,164 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:07:29,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:29,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:07:30,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:32,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:34,048 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:36,049 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:41,405 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:07:41,405 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:07:48,205 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:07:48,206 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:07:48,206 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:07:49,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:49,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:07:50,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:52,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:54,056 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:56,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:07:56,534 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:07:56,534 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:07:57,025 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:08:08,246 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:08:08,246 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:08:08,247 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:08:09,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:09,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:08:10,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:11,670 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:08:11,671 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:08:12,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:14,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:16,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:26,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:08:26,802 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:08:27,103 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:08:28,291 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:08:28,291 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:08:28,292 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:08:29,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:29,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:08:30,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:32,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:34,073 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:36,074 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:41,940 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:08:41,941 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:08:48,337 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:08:48,338 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:08:48,338 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:08:49,078 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:08:50,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:52,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:54,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:56,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:08:57,071 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:08:57,071 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:08:57,200 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:09:08,376 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:09:08,376 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:09:08,377 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:09:09,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:09:10,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:12,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:12,201 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:09:12,202 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:09:14,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:16,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:27,253 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:09:27,333 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:09:27,333 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:09:28,413 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:09:28,414 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:09:28,414 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:09:29,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:09:30,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:32,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:34,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:36,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:42,461 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:09:42,462 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:09:48,452 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:09:48,452 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:09:48,453 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:09:49,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:09:50,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:52,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:54,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:56,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:09:57,331 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:09:57,593 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:09:57,593 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:10:08,489 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:10:08,489 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:10:08,490 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:10:09,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:10:10,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:12,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:12,727 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:10:12,727 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:10:14,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:16,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:27,409 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:10:27,859 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:10:27,859 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:10:28,526 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:10:28,527 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:10:28,527 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:10:29,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:10:30,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:32,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:34,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:36,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:42,991 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:10:42,991 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:10:48,569 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:10:48,569 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:10:48,570 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:10:49,128 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:10:50,128 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:52,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:54,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:56,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:10:57,488 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:10:58,123 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:10:58,123 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:11:08,607 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:11:08,607 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:11:08,608 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:11:09,136 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:11:10,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:12,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:13,252 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:11:13,253 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:11:14,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:16,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:27,565 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:11:28,384 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:11:28,384 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:11:28,645 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:11:28,645 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:11:28,646 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:11:29,144 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:11:30,145 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:32,145 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:34,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:36,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:43,526 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:11:43,527 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:11:48,684 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:11:48,685 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:11:48,685 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:11:49,152 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:11:50,153 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:52,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:54,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:56,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:11:57,643 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:11:58,656 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:11:58,657 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:12:08,722 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:12:08,722 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:12:08,723 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:12:09,161 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:12:10,161 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:11,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:13,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:13,786 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:12:13,786 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:12:15,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:17,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:27,720 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:12:28,759 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:12:28,759 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:12:28,760 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:12:28,932 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:12:28,932 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:12:29,169 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:12:30,170 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:31,170 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:33,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:35,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:37,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:44,063 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:12:44,063 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:12:48,798 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:12:48,798 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:12:48,799 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:12:49,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:12:50,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:51,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:53,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:55,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:57,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:12:57,798 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:12:59,194 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:12:59,194 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:13:08,837 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:13:08,838 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:13:08,838 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:13:09,186 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:13:10,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:11,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:13,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:14,323 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:13:14,324 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:13:15,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:17,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:27,875 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:13:28,874 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:13:28,875 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:13:28,875 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:13:29,195 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:13:29,456 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:13:29,456 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:13:30,196 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:31,196 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:33,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:35,198 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:37,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:44,595 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:13:44,595 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:13:48,912 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:13:48,912 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:13:48,912 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:13:49,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:13:50,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:51,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:53,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:55,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:57,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:13:57,953 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:13:59,727 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:13:59,727 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:14:08,951 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:14:08,952 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:14:08,952 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:14:09,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:14:10,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:11,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:13,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:14,857 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:14:14,857 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:14:15,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:17,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:28,031 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:14:29,018 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:14:29,019 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:14:29,019 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:14:29,222 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:14:29,994 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:14:29,994 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:14:30,223 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:31,223 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:33,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:35,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:37,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:45,160 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:14:45,160 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:14:49,063 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:14:49,064 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:14:49,064 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:14:49,231 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:14:50,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:51,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:53,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:55,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:57,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:14:58,109 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:15:00,294 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:15:00,295 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:15:09,112 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:15:09,112 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:15:09,113 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:15:09,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:15:10,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:11,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:13,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:15,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:15,426 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:15:15,426 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:15:17,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:28,186 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:15:29,175 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:15:29,175 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:15:29,176 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:15:29,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:15:30,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:30,565 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:15:30,565 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:15:31,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:33,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:35,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:37,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:45,736 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:15:45,736 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:15:49,214 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:15:49,215 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:15:49,215 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:15:49,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:15:50,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:51,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:53,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:55,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:57,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:15:58,264 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:16:00,869 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:16:00,869 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:16:09,254 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:16:09,254 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:16:09,255 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:16:09,266 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:16:10,266 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:11,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:13,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:15,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:16,002 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:16:16,003 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:16:17,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:28,345 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:16:29,298 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:16:29,298 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:16:29,299 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:16:30,275 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:30,275 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:16:31,144 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:16:31,144 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:16:31,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:33,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:35,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:37,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:46,308 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:16:46,308 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:16:49,335 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:16:49,336 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:16:49,336 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:16:50,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:50,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:16:51,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:53,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:55,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:57,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:16:58,424 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:17:01,439 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:17:01,439 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:17:09,376 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:17:09,377 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:17:09,377 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:17:10,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:10,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:17:11,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:13,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:15,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:16,596 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:17:16,596 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:17:17,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:28,503 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:17:29,415 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:17:29,416 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:17:29,416 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:17:30,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:30,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:17:31,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:31,764 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:17:31,764 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:17:33,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:35,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:37,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:46,894 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:17:46,895 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:17:49,455 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:17:49,455 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:17:49,456 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:17:50,310 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:50,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:17:51,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:53,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:55,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:57,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:17:58,580 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:18:02,027 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:18:02,028 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:18:09,494 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:18:09,494 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:18:09,495 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:18:10,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:10,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:18:11,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:13,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:15,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:17,160 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:18:17,161 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:18:17,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:28,658 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:18:29,531 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:18:29,532 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:18:29,532 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:18:30,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:18:31,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:32,292 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:18:32,292 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:18:33,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:35,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:37,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:47,435 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:18:47,435 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:18:49,569 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:18:49,569 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:18:49,570 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:18:50,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:18:51,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:53,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:55,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:57,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:18:58,736 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:19:02,565 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:19:02,566 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:19:09,606 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:19:09,606 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:19:09,607 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:19:10,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:19:11,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:13,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:15,347 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:17,348 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:17,698 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:19:17,698 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:19:28,813 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:19:29,644 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:19:29,644 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:19:29,645 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:19:30,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:19:31,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:32,827 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:19:32,827 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:19:33,355 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:35,356 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:37,356 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:47,961 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:19:47,962 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:19:49,682 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:19:49,683 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:19:49,683 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:19:50,362 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:19:51,363 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:53,363 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:55,364 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:57,365 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:19:58,891 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:20:03,101 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:20:03,101 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:20:09,719 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:20:09,719 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:20:09,720 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:20:10,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:20:11,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:13,372 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:15,373 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:17,374 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:18,233 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:20:18,234 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:20:28,968 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:20:29,761 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:20:29,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:20:29,762 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:20:30,381 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:20:31,381 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:33,378 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:20:33,378 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:20:33,382 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:35,383 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:37,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:48,508 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:20:48,508 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:20:49,797 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:20:49,797 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:20:49,805 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:20:50,389 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:20:51,389 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:53,390 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:55,391 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:57,392 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:20:59,043 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:21:03,643 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:21:03,644 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:21:09,833 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:21:09,833 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:21:09,834 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:21:10,398 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:21:11,398 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:13,399 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:15,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:17,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:18,778 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:21:18,779 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:21:29,119 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:21:29,869 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:21:29,869 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:21:29,870 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:21:30,406 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:21:31,406 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:33,407 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:33,917 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:21:33,918 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:21:35,408 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:37,409 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:49,050 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:21:49,050 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:21:49,907 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:21:49,908 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:21:49,908 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:21:50,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:21:51,415 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:52,415 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:54,416 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:56,417 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:58,418 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:21:59,197 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:22:04,185 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:22:04,186 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:22:09,944 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:22:09,945 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:22:09,945 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:22:10,423 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:22:11,423 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:12,424 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:14,425 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:16,426 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:18,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:19,317 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:22:19,317 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:22:29,274 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:22:29,983 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:22:29,983 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:22:29,984 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:22:30,432 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:22:31,432 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:32,433 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:34,434 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:34,476 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:22:34,477 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:22:36,434 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:38,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:49,612 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:22:49,612 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:22:50,022 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:22:50,022 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:22:50,023 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:22:50,441 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:22:51,441 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:52,441 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:54,442 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:56,443 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:58,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:22:59,351 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:23:04,746 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:23:04,746 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:23:10,069 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:23:10,070 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:23:10,070 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:23:10,449 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:23:11,449 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:12,450 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:14,451 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:16,451 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:18,452 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:19,878 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:23:19,879 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:23:29,430 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:23:30,108 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:23:30,109 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:23:30,109 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:23:30,457 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:23:31,458 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:32,458 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:34,459 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:35,021 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:23:35,021 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:23:36,460 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:38,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:50,146 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:23:50,146 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:23:50,147 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:23:50,222 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:23:50,222 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:23:50,466 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:23:51,466 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:52,466 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:54,467 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:56,468 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:58,469 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:23:59,508 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:24:05,387 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:24:05,387 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:24:10,190 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:24:10,191 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:24:10,191 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:24:10,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:24:11,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:12,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:14,476 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:16,477 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:18,478 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:20,518 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:24:20,518 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:24:29,585 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:24:30,230 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:24:30,230 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:24:30,231 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:24:30,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:24:31,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:32,484 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:34,485 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:35,649 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:24:35,649 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:24:36,486 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:38,486 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:50,270 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:24:50,270 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:24:50,271 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:24:50,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:24:50,826 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:24:50,827 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:24:51,492 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:52,492 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:54,493 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:56,494 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:58,495 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:24:59,663 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:25:06,076 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:25:06,076 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:25:10,311 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:25:10,312 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:25:10,312 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:25:10,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:25:11,501 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:12,501 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:14,502 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:16,503 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:18,503 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:21,208 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:25:21,208 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:25:29,741 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:25:30,351 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:25:30,351 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:25:30,351 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:25:30,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:25:31,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:32,510 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:34,511 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:36,341 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:25:36,341 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:25:36,511 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:38,512 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:50,392 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:25:50,392 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:25:50,393 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:25:50,518 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:25:51,480 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:25:51,480 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:25:51,518 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:52,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:54,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:56,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:58,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:25:59,820 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:26:06,611 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:26:06,612 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:26:10,431 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:26:10,431 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:26:10,432 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:26:10,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:26:11,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:12,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:14,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:16,529 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:18,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:21,747 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:26:21,747 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:26:29,897 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:26:30,482 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:26:30,482 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:26:30,483 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:26:30,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:26:31,536 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:32,536 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:34,537 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:36,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:36,883 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:26:36,884 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:26:38,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:50,517 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:26:50,517 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:26:50,518 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:26:50,544 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:26:51,545 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:52,017 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:26:52,018 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:26:52,545 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:54,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:56,547 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:58,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:26:59,975 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:27:07,159 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:27:07,159 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:27:10,553 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:27:10,553 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:27:10,554 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:27:11,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:11,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:27:12,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:14,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:16,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:18,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:22,292 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:27:22,292 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:27:30,052 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:27:30,597 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:27:30,597 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:27:30,598 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:27:31,563 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:31,563 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:27:32,563 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:34,564 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:36,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:37,433 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:27:37,433 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:27:38,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:50,633 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:27:50,634 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:27:50,634 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:27:51,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:51,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:27:52,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:52,596 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:27:52,597 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:27:54,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:56,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:27:58,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:00,130 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:28:07,738 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:28:07,739 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:28:10,672 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:28:10,672 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:28:10,672 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:28:11,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:11,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:28:12,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:14,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:16,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:18,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:22,872 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:28:22,872 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:28:30,208 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:28:30,707 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:28:30,707 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:28:30,707 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:28:31,588 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:31,589 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:28:32,589 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:34,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:36,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:38,005 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:28:38,005 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:28:38,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:50,744 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:28:50,745 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:28:50,745 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:28:51,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:51,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:28:52,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:53,151 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:28:53,151 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:28:54,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:56,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:28:58,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:00,286 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:29:08,283 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:29:08,284 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:29:10,785 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:29:10,785 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:29:10,785 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:29:11,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:11,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:29:12,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:14,608 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:16,608 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:18,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:23,415 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:29:23,415 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:29:30,364 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:29:30,822 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:29:30,822 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:29:30,822 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:29:31,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:29:32,616 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:34,616 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:36,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:38,546 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:29:38,546 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:29:38,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:50,860 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:29:50,860 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:29:50,861 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:29:51,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:29:52,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:53,685 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:29:53,685 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:29:54,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:56,626 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:29:58,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:00,441 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:30:08,816 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:30:08,816 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:30:10,899 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:30:10,899 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:30:10,900 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:30:11,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:30:12,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:14,634 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:16,634 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:18,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:23,955 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:30:23,956 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:30:30,518 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:30:30,943 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:30:30,944 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:30:30,944 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:30:31,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:30:32,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:34,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:36,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:38,644 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:39,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:30:39,089 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:30:50,983 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:30:50,983 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:30:50,984 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:30:51,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:30:52,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:54,220 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:30:54,221 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:30:54,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:56,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:30:58,652 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:00,596 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:31:09,352 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:31:09,353 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:31:11,022 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:31:11,022 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:31:11,022 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:31:11,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:31:12,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:14,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:16,659 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:18,660 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:24,490 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:31:24,491 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:31:30,673 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:31:31,064 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:31:31,065 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:31:31,065 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:31:31,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:31:32,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:34,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:36,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:38,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:39,622 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:31:39,622 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:31:40,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:42,669 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:44,670 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:46,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:49,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:51,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:53,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:54,768 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:31:54,768 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:31:55,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:57,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:31:59,676 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:00,746 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:32:01,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:03,678 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:05,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:07,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:09,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:09,904 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:32:09,904 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:32:11,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:13,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:15,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:17,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:19,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:21,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:23,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:25,039 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:32:25,039 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:32:25,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:27,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:29,690 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:30,817 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:32:31,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:35,031 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:32:35,031 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:32:35,032 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:32:35,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:32:37,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:39,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:40,170 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:32:40,170 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:32:41,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:43,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:55,086 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:32:55,086 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:32:55,086 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:32:55,334 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:32:55,334 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:32:55,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:32:56,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:57,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:32:59,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:00,886 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:33:01,704 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:03,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:10,474 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:33:10,475 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:33:15,151 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:33:15,152 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:33:15,152 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:33:15,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:33:16,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:17,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:19,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:21,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:23,714 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:25,606 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:33:25,606 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:33:30,954 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:33:35,192 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:33:35,193 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:33:35,193 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:33:35,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:35,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:33:36,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:37,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:39,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:40,736 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:33:40,737 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:33:41,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:43,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:55,237 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:33:55,238 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:33:55,238 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:33:55,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:55,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:33:55,923 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:33:55,923 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:33:56,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:57,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:33:59,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:01,028 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:34:01,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:03,732 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:11,064 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:34:11,064 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:34:15,276 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:34:15,277 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:34:15,277 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:34:15,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:34:16,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:17,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:19,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:21,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:23,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:26,195 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:34:26,196 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:34:31,096 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:34:35,324 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:34:35,325 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:34:35,325 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:34:35,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:34:36,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:37,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:39,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:41,328 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:34:41,328 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:34:41,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:43,749 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:55,370 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:34:55,370 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:34:55,371 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:34:55,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:55,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:34:56,466 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:34:56,466 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:34:56,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:57,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:34:59,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:01,165 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:35:01,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:03,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:11,628 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:35:11,628 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:35:15,413 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:35:15,413 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:35:15,414 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:35:15,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:15,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:35:16,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:17,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:19,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:21,765 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:23,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:26,758 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:35:26,759 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:35:31,232 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:35:35,456 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:35:35,457 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:35:35,457 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:35:35,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:35,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:35:36,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:38,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:40,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:41,891 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:35:41,891 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:35:42,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:44,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:55,512 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:35:55,512 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:35:55,513 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:35:55,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:35:56,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:35:57,024 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:35:57,024 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:35:58,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:00,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:01,303 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:36:02,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:04,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:12,214 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:36:12,215 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:36:15,561 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:36:15,561 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:36:15,562 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:36:15,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:36:16,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:17,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:18,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:20,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:22,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:24,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:27,356 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:36:27,357 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:36:31,373 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:36:35,600 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:36:35,600 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:36:35,601 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:36:35,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:36:37,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:38,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:40,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:42,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:36:42,488 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:36:42,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:44,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:55,639 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:36:55,639 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:36:55,640 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:36:55,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:36:57,629 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:36:57,629 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:36:57,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:36:58,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:00,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:01,444 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:37:02,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:04,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:12,769 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:37:12,770 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:37:15,685 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:37:15,685 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:37:15,686 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:37:15,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:37:17,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:18,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:20,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:22,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:24,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:27,901 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:37:27,902 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:37:31,518 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:37:35,723 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:37:35,723 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:37:35,724 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:37:35,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:37:36,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:37,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:38,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:40,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:42,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:43,037 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:37:43,038 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:37:44,827 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:55,762 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:37:55,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:37:55,763 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:37:55,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:37:56,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:57,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:37:58,175 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:37:58,176 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:37:58,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:00,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:01,592 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:38:02,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:04,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:13,312 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:38:13,312 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:38:15,801 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:38:15,801 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:38:15,802 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:38:15,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:38:16,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:17,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:18,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:20,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:22,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:24,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:28,444 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:38:28,445 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:38:31,663 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:38:35,843 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:38:35,844 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:38:35,844 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:38:35,849 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:38:36,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:37,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:38,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:40,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:42,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:43,580 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:38:43,580 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:38:44,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:55,889 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:38:55,890 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:38:55,890 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:38:56,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:38:57,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:38:58,711 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:38:58,711 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:38:58,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:00,860 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:01,734 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:39:02,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:04,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:13,842 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:39:13,843 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:39:15,931 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:39:15,931 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:39:15,932 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:39:16,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:16,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:39:17,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:18,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:20,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:22,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:24,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:28,979 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:39:28,979 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:39:31,808 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:39:35,969 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:39:35,969 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:39:35,970 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:39:36,875 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:36,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:39:37,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:38,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:40,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:42,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:44,114 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:39:44,114 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:39:44,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:56,014 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:39:56,015 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:39:56,015 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:39:56,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:56,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:39:57,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:58,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:39:59,244 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:39:59,244 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:40:00,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:01,883 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:40:02,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:04,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:14,431 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:40:14,432 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:40:16,058 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:40:16,058 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:40:16,059 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:40:16,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:16,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:40:17,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:18,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:20,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:22,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:24,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:29,570 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:40:29,570 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:40:31,958 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:40:36,096 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:40:36,097 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:40:36,097 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:40:36,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:36,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:40:37,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:38,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:40,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:42,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:44,701 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:40:44,702 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:40:44,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:56,373 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:40:56,374 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:40:56,374 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:40:56,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:56,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:40:57,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:58,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:40:59,839 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:40:59,839 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:41:00,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:02,033 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:41:02,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:04,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:14,970 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:41:14,971 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:41:16,412 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:41:16,412 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:41:16,413 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:41:16,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:16,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:41:17,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:18,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:20,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:22,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:24,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:30,110 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:41:30,110 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:41:32,107 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:41:36,451 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:41:36,451 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:41:36,452 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:41:36,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:36,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:41:37,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:38,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:40,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:42,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:44,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:45,242 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:41:45,243 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:41:56,490 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:41:56,490 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:41:56,491 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:41:56,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:56,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:41:57,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:41:58,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:00,375 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:42:00,375 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:42:00,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:02,185 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:42:02,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:04,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:15,506 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:42:15,506 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:42:16,531 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:42:16,531 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:42:16,532 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:42:16,944 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:42:17,944 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:19,945 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:21,946 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:23,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:25,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:30,638 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:42:30,639 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:42:32,262 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:42:36,574 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:42:36,574 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:42:36,574 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:42:36,953 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:42:37,953 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:39,954 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:41,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:43,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:45,770 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:42:45,771 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:42:45,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:56,611 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:42:56,612 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:42:56,612 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:42:56,962 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:42:57,962 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:42:59,963 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:00,930 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:43:00,931 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:43:01,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:02,335 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:43:03,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:05,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:16,069 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:43:16,069 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:43:16,648 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:43:16,649 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:43:16,649 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:43:16,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:43:17,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:19,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:21,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:23,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:25,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:31,200 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:43:31,200 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:43:32,409 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:43:36,686 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:43:36,687 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:43:36,687 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:43:36,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:43:37,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:39,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:41,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:43,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:45,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:46,330 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:43:46,330 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:43:56,724 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:43:56,725 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:43:56,725 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:43:56,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:43:57,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:58,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:43:59,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:01,461 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:44:01,462 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:44:01,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:02,484 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:44:03,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:05,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:16,618 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:44:16,618 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:44:16,762 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:44:16,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:44:16,763 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:44:16,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:44:17,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:18,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:19,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:22,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:24,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:26,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:31,751 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:44:31,752 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:44:32,559 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:44:36,801 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:44:36,801 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:44:36,802 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:44:37,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:44:38,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:39,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:40,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:42,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:44,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:46,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:46,885 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:44:46,885 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:44:56,842 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:44:56,842 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:44:56,843 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:44:57,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:44:58,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:44:59,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:00,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:02,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:02,022 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:45:02,023 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:45:02,632 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:45:04,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:06,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:16,880 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:45:16,880 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:45:16,881 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:45:17,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:45:17,184 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:45:17,184 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:45:18,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:19,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:20,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:22,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:24,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:26,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:32,316 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:45:32,316 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:45:32,708 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:45:36,924 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:45:36,925 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:45:36,925 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:45:37,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:45:38,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:39,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:40,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:42,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:44,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:46,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:47,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:45:47,449 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:45:56,966 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:45:56,967 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:45:56,967 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:45:57,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:45:58,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:45:59,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:00,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:02,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:02,578 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:46:02,578 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:46:02,783 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:46:04,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:06,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:17,003 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:46:17,004 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:46:17,004 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:46:17,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:46:17,729 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:46:17,729 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:46:18,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:19,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:20,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:22,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:24,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:26,056 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:32,857 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:46:32,859 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:46:32,860 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:46:37,044 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:46:37,044 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:46:37,044 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:46:37,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:46:38,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:39,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:40,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:42,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:44,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:46,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:47,992 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:46:47,993 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:46:57,087 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:46:57,087 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:46:57,088 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:46:58,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:46:58,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:46:59,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:00,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:02,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:02,931 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:47:03,140 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:47:03,140 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:47:04,073 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:06,074 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:17,128 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:47:17,128 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:47:17,129 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:47:18,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:18,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:47:18,280 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:47:18,280 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:47:19,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:20,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:22,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:24,082 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:26,082 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:33,006 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:47:33,411 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:47:33,411 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:47:37,168 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:47:37,169 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:47:37,169 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:47:38,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:38,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:47:39,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:40,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:42,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:44,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:46,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:48,541 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:47:48,541 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:47:57,209 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:47:57,210 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:47:57,210 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:47:58,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:47:58,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:47:59,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:00,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:02,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:03,081 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:48:03,673 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:48:03,674 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:48:04,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:06,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:17,483 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:48:17,483 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:48:17,484 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:48:18,103 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:18,103 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:48:18,829 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:48:18,829 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:48:19,103 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:20,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:22,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:24,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:26,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:33,156 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:48:33,969 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:48:33,970 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:48:37,522 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:48:37,522 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:48:37,523 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:48:38,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:38,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:48:39,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:40,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:42,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:44,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:46,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:49,110 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:48:49,111 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:48:57,561 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:48:57,562 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:48:57,562 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:48:58,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:48:58,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:48:59,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:00,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:02,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:03,230 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:49:04,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:04,241 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:49:04,242 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:49:06,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:17,599 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:49:17,599 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:49:17,599 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:49:18,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:49:19,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:19,399 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:49:19,399 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:49:21,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:23,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:25,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:27,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:33,303 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:49:34,538 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:49:34,539 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:49:37,637 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:49:37,637 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:49:37,638 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:49:38,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:49:39,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:41,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:43,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:45,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:47,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:49:49,667 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:49:49,667 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:49:57,674 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:49:57,675 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:49:57,675 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:49:58,145 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:49:59,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:01,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:03,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:03,378 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:50:04,799 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:50:04,799 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:50:05,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:07,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:17,711 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:50:17,712 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:50:17,712 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:50:18,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:50:19,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:19,959 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:50:19,960 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:50:21,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:23,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:25,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:27,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:33,449 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:50:35,101 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:50:35,102 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:50:37,748 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:50:37,748 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:50:37,748 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:50:38,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:50:39,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:41,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:43,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:45,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:47,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:50:50,232 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:50:50,232 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:50:57,792 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:50:57,793 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:50:57,793 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:50:58,170 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:50:59,170 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:01,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:03,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:03,519 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:51:05,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:05,365 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:51:05,366 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:51:07,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:17,829 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:51:17,830 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:51:17,830 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:51:18,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:51:19,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:20,496 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:51:20,496 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:51:21,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:23,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:25,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:27,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:33,595 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:51:35,626 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:51:35,626 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:51:37,868 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:51:37,868 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:51:37,868 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:51:38,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:51:39,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:41,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:43,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:45,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:47,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:51:50,756 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:51:50,757 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:51:57,905 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:51:57,905 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:51:57,906 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:51:58,195 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:51:59,196 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:00,196 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:01,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:03,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:03,669 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:52:05,198 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:05,888 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:52:05,888 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:52:07,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:17,951 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:52:17,951 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:52:17,951 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:52:18,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:52:19,204 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:20,204 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:21,016 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:52:21,017 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:52:21,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:23,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:25,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:27,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:33,743 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:52:36,150 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:52:36,151 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:52:37,994 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:52:37,994 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:52:37,995 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:52:38,212 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:52:39,212 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:40,213 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:45,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:49,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:51,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:52:51,283 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:52:51,283 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:53:03,821 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:53:06,414 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:53:06,519 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status