diff --git "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" --- "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" +++ "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" @@ -9439,3 +9439,3358 @@ 2021-07-15 23:22:39,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json 2021-07-15 23:22:40,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log 2021-07-15 23:22:42,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:22:44,129 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:22:46,130 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:22:48,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:22:51,523 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:22:51,523 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:22:56,782 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:22:58,468 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:22:58,468 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:22:58,469 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:22:59,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:23:00,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:02,136 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:04,137 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:06,138 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:06,655 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:23:06,655 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:23:08,139 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:18,508 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:23:18,508 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:23:18,509 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:23:19,144 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:23:20,144 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:21,809 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:23:21,810 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:23:22,145 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:24,145 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:26,146 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:26,854 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:23:28,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:36,942 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:23:36,942 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:23:38,550 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:23:38,550 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:23:38,551 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:23:39,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:23:40,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:42,152 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:44,153 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:46,154 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:23:52,073 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:23:52,074 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:23:56,926 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:23:58,590 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:23:58,590 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:23:58,591 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:23:59,158 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:24:00,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:02,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:04,160 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:06,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:07,206 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:24:07,207 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:24:08,162 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:18,645 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:24:18,645 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:24:18,646 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:24:19,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:24:20,166 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:22,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:22,343 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:24:22,343 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:24:24,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:26,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:26,997 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:24:28,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:37,476 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:24:37,476 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:24:38,683 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:24:38,684 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:24:38,684 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:24:39,172 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:24:40,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:42,174 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:44,174 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:46,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:48,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:24:52,611 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:24:52,612 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:24:57,071 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:24:58,731 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:24:58,731 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:24:58,732 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:24:59,179 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:25:00,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:02,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:04,181 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:06,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:07,954 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:25:07,955 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:25:08,183 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:18,770 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:25:18,771 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:25:18,771 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:25:19,186 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:25:20,186 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:22,187 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:23,093 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:25:23,093 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:25:24,188 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:26,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:27,147 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:25:38,233 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:25:38,233 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:25:38,811 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:25:38,811 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:25:38,812 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:25:39,193 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:25:40,194 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:42,194 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:44,195 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:46,196 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:25:53,364 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:25:53,364 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:25:57,221 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:25:58,849 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:25:58,849 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:25:58,850 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:25:59,200 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:26:00,201 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:02,201 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:04,202 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:06,203 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:08,495 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:26:08,495 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:26:18,886 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:26:18,886 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:26:18,887 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:26:19,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:26:20,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:21,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:22,209 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:23,628 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:26:23,628 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:26:24,209 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:26,210 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:27,298 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:26:28,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:38,799 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:26:38,800 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:26:38,928 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:26:38,929 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:26:38,929 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:26:39,214 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:26:40,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:41,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:42,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:44,216 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:46,217 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:48,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:26:53,962 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:26:53,963 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:26:57,373 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:26:58,959 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:26:58,960 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:26:58,960 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:26:59,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:27:00,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:01,223 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:02,223 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:04,224 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:06,225 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:08,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:09,096 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:27:09,096 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:27:19,016 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:27:19,016 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:27:19,017 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:27:19,230 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:27:20,231 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:21,231 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:22,232 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:24,229 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:27:24,230 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:27:24,232 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:26,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:27,444 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:27:28,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:39,056 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:27:39,056 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:27:39,056 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:27:39,238 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:27:39,375 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:27:39,376 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:27:40,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:41,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:42,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:44,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:46,241 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:48,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:27:54,507 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:27:54,507 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:27:57,520 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:27:59,093 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:27:59,094 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:27:59,094 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:27:59,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:28:00,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:01,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:02,247 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:04,247 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:06,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:08,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:09,639 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:28:09,639 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:28:19,137 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:28:19,137 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:28:19,138 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:28:19,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:28:20,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:21,254 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:22,254 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:24,255 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:24,771 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:28:24,771 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:28:26,255 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:27,594 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:28:39,184 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:28:39,185 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:28:39,185 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:28:39,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:28:39,919 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:28:39,919 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:28:40,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:41,261 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:42,261 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:44,262 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:46,263 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:28:55,052 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:28:55,052 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:28:57,667 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:28:59,223 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:28:59,223 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:28:59,224 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:28:59,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:29:00,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:01,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:02,269 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:04,269 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:06,270 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:08,271 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:10,182 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:29:10,182 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:29:19,261 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:29:19,261 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:29:19,261 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:29:19,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:29:20,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:21,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:22,276 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:24,277 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:25,311 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:29:25,311 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:29:26,277 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:27,740 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:29:28,278 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:39,306 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:29:39,306 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:29:39,306 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:29:40,282 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:29:40,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:40,443 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:29:40,443 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:29:41,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:42,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:44,284 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:46,284 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:48,285 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:29:55,584 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:29:55,585 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:29:57,814 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:29:59,343 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:29:59,344 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:29:59,344 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:30:00,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:30:00,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:01,290 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:03,290 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:05,291 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:07,292 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:09,293 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:10,716 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:30:10,716 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:30:19,379 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:30:19,380 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:30:19,380 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:30:20,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:30:21,297 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:23,297 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:25,298 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:25,912 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:30:25,913 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:30:27,299 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:27,887 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:30:29,300 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:39,430 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:30:39,430 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:30:39,431 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:30:40,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:30:41,050 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:30:41,051 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:30:41,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:43,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:45,305 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:47,306 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:49,307 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:30:56,184 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:30:56,185 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:30:57,962 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:30:59,468 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:30:59,469 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:30:59,469 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:31:00,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:31:01,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:03,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:05,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:07,313 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:09,314 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:11,318 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:31:11,318 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:31:19,507 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:31:19,507 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:31:19,507 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:31:20,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:31:21,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:23,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:25,320 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:26,449 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:31:26,449 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:31:27,321 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:28,037 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:31:29,322 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:39,544 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:31:39,545 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:31:39,545 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:31:40,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:31:41,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:41,582 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:31:41,582 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:31:43,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:45,328 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:47,329 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:31:56,755 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:31:56,755 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:31:58,109 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:31:59,585 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:31:59,585 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:31:59,586 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:32:00,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:32:01,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:03,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:05,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:07,336 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:11,888 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:32:11,888 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:32:19,625 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:32:19,625 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:32:19,625 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:32:20,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:32:21,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:23,341 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:25,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:27,021 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:32:27,021 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:32:27,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:28,185 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:32:39,663 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:32:39,663 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:32:39,664 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:32:40,347 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:32:41,348 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:42,186 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:32:42,186 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:32:43,348 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:45,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:47,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:49,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:32:57,324 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:32:57,325 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:32:58,259 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:32:59,703 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:32:59,703 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:32:59,704 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:33:00,355 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:33:01,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:03,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:05,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:07,358 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:09,358 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:12,455 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:33:12,456 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:33:19,752 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:33:19,752 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:33:19,753 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:33:20,362 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:33:21,362 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:23,363 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:25,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:27,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:27,585 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:33:27,585 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:33:28,329 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:33:29,365 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:31,366 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:33,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:35,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:37,368 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:39,369 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:41,370 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:42,717 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:33:42,717 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:33:43,370 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:45,371 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:47,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:49,373 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:51,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:53,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:55,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:57,376 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:33:57,881 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:33:57,881 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:33:58,401 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:33:59,377 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:01,378 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:03,379 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:05,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:07,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:09,381 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:11,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:13,011 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:34:13,011 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:34:13,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:15,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:17,384 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:20,385 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:22,386 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:23,586 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:34:23,586 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:34:23,586 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:34:24,386 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:34:26,387 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:28,143 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:34:28,143 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:34:28,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:28,473 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:34:30,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:32,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:43,272 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:34:43,272 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:34:43,643 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:34:43,644 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:34:43,644 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:34:44,394 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:34:46,395 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:48,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:50,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:52,397 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:34:58,405 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:34:58,405 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:34:58,544 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:35:03,682 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:35:03,682 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:35:03,682 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:35:04,402 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:35:06,403 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:08,403 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:10,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:12,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:13,535 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:35:13,535 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:35:23,746 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:35:23,747 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:35:23,747 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:35:24,409 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:35:26,410 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:28,411 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:28,616 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:35:28,665 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:35:28,666 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:35:30,412 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:32,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:43,783 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:35:43,784 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:35:43,784 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:35:43,822 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:35:43,822 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:35:44,417 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:35:46,418 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:48,419 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:50,420 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:52,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:35:58,688 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:35:58,976 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:35:58,977 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:36:03,818 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:36:03,819 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:36:03,819 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:36:04,426 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:36:06,427 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:08,428 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:10,428 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:12,429 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:14,107 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:36:14,107 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:36:23,891 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:36:23,892 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:36:23,892 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:36:24,433 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:36:26,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:28,435 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:28,759 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:36:29,240 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:36:29,240 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:36:30,436 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:32,437 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:43,892 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:36:43,893 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:36:43,893 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:36:44,370 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:36:44,370 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:36:44,441 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:36:46,442 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:48,443 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:50,443 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:52,444 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:36:58,829 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:36:59,509 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:36:59,510 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:37:03,946 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:37:03,946 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:37:03,946 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:37:04,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:37:06,449 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:08,450 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:10,451 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:12,452 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:14,639 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:37:14,640 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:37:23,984 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:37:23,984 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:37:23,985 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:37:24,457 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:37:26,458 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:28,459 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:28,900 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:37:29,771 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:37:29,771 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:37:30,460 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:32,461 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:44,028 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:37:44,028 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:37:44,028 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:37:44,465 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:37:44,913 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:37:44,914 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:37:46,466 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:48,467 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:50,468 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:52,469 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:37:58,970 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:38:00,044 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:38:00,044 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:38:04,066 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:38:04,066 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:38:04,067 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:38:04,473 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:38:06,474 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:08,475 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:10,475 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:12,476 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:15,174 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:38:15,174 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:38:24,111 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:38:24,112 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:38:24,112 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:38:24,481 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:38:26,482 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:28,482 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:29,041 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:38:30,303 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:38:30,304 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:38:30,483 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:32,484 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:44,187 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:38:44,188 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:38:44,188 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:38:44,489 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:38:45,479 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:38:45,479 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:38:46,490 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:48,491 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:50,491 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:52,492 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:38:59,111 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:39:00,636 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:39:00,637 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:39:04,244 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:39:04,245 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:39:04,245 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:39:04,497 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:39:06,498 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:08,498 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:10,499 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:12,500 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:15,770 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:39:15,771 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:39:24,283 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:39:24,283 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:39:24,284 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:39:24,505 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:39:26,506 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:28,507 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:29,178 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:39:30,508 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:30,900 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:39:30,900 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:39:32,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:44,336 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:39:44,337 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:39:44,337 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:39:44,514 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:39:46,048 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:39:46,049 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:39:46,515 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:48,516 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:50,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:52,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:39:59,251 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:40:01,179 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:40:01,180 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:40:04,386 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:40:04,386 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:40:04,387 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:40:04,522 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:40:06,523 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:08,523 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:10,524 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:12,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:16,311 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:40:16,311 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:40:24,434 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:40:24,435 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:40:24,435 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:40:24,530 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:40:26,530 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:28,531 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:29,324 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:40:30,532 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:31,452 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:40:31,453 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:40:32,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:44,473 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:40:44,474 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:40:44,474 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:40:44,537 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:40:46,538 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:46,582 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:40:46,583 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:40:48,539 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:50,539 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:52,540 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:40:59,399 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:41:01,721 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:41:01,721 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:41:04,515 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:41:04,515 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:41:04,516 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:41:04,545 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:41:06,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:08,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:10,547 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:12,548 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:16,850 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:41:16,851 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:41:24,555 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:41:24,556 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:41:24,556 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:41:25,553 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:41:26,553 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:28,554 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:29,475 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:41:30,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:31,982 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:41:31,982 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:41:32,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:44,628 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:41:44,629 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:41:44,629 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:41:45,560 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:41:46,560 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:47,124 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:41:47,125 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:41:48,561 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:50,562 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:52,563 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:41:59,550 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:42:02,263 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:42:02,263 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:42:04,665 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:42:04,666 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:42:04,666 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:42:05,568 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:42:06,568 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:08,569 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:10,570 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:12,571 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:17,393 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:42:17,393 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:42:24,706 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:42:24,706 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:42:24,707 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:42:25,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:42:26,576 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:28,576 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:29,623 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:42:30,577 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:32,522 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:42:32,522 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:42:32,578 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:44,744 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:42:44,744 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:42:44,745 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:42:45,583 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:42:46,583 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:47,664 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:42:47,665 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:42:48,584 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:50,585 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:52,586 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:42:59,691 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:43:02,795 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:43:02,796 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:43:04,782 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:43:04,783 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:43:04,783 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:43:05,591 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:43:06,591 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:08,592 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:10,593 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:12,594 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:17,930 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:43:17,930 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:43:24,821 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:43:24,822 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:43:24,822 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:43:25,599 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:43:26,599 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:28,600 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:29,757 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:43:30,601 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:32,601 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:33,062 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:43:33,063 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:43:44,867 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:43:44,867 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:43:44,867 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:43:45,606 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:43:46,607 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:48,198 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:43:48,199 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:43:48,607 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:50,608 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:52,609 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:43:59,821 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:44:03,330 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:44:03,330 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:44:04,936 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:44:04,936 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:44:04,937 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:44:05,614 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:44:07,615 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:09,616 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:11,616 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:13,617 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:18,464 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:44:18,465 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:44:24,975 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:44:24,976 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:44:24,976 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:44:25,622 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:44:27,623 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:29,623 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:29,886 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:44:31,624 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:33,597 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:44:33,597 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:44:33,625 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:45,015 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:44:45,015 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:44:45,016 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:44:45,630 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:44:47,630 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:48,728 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:44:48,729 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:44:49,631 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:51,632 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:53,633 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:44:59,951 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:45:03,859 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:45:03,859 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:45:05,054 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:45:05,055 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:45:05,055 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:45:05,637 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:45:07,638 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:09,639 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:11,639 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:13,640 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:18,992 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:45:18,992 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:45:25,113 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:45:25,113 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:45:25,114 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:45:25,644 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:45:27,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:29,646 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:30,016 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:45:31,647 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:33,648 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:34,124 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:45:34,124 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:45:45,154 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:45:45,154 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:45:45,155 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:45:45,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:45:47,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:49,265 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:45:49,265 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:45:49,654 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:51,654 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:45:53,655 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:00,080 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:46:04,399 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:46:04,400 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:46:05,228 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:46:05,228 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:46:05,229 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:46:05,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:46:07,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:09,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:11,662 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:13,662 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:19,534 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:46:19,534 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:46:25,268 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:46:25,268 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:46:25,268 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:46:25,668 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:46:27,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:29,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:30,149 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:46:31,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:33,671 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:34,667 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:46:34,667 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:46:45,269 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:46:45,270 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:46:45,270 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:46:45,676 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:46:47,676 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:49,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:49,801 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:46:49,801 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:46:51,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:46:53,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:00,216 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:47:04,934 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:47:04,934 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:47:05,309 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:47:05,309 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:47:05,310 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:47:05,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:47:07,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:09,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:11,688 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:13,688 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:20,063 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:47:20,064 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:47:25,349 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:47:25,349 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:47:25,350 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:47:25,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:47:27,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:29,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:30,284 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:47:31,695 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:33,696 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:35,195 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:47:35,196 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:47:45,390 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:47:45,390 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:47:45,391 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:47:45,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:47:47,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:49,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:50,324 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:47:50,324 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:47:51,703 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:47:53,704 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:00,351 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:48:05,458 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:48:05,458 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:48:05,458 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:48:05,459 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:48:05,459 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:48:05,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:48:07,709 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:09,710 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:11,711 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:13,712 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:20,588 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:48:20,589 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:48:25,497 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:48:25,497 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:48:25,498 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:48:25,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:48:27,717 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:29,718 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:30,417 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:48:31,719 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:33,719 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:35,720 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:48:35,721 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:48:45,547 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:48:45,547 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:48:45,548 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:48:45,724 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:48:47,725 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:49,726 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:50,853 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:48:50,853 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:48:51,726 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:48:53,727 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:00,484 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:49:05,585 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:49:05,586 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:49:05,586 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:49:05,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:49:05,983 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:49:05,983 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:49:07,732 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:09,733 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:11,734 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:13,735 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:21,141 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:49:21,141 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:49:25,651 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:49:25,652 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:49:25,652 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:49:25,740 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:49:27,740 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:29,741 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:30,554 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:49:31,742 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:33,743 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:36,272 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:49:36,272 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:49:45,691 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:49:45,691 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:49:45,692 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:49:45,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:49:47,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:49,748 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:51,404 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:49:51,404 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:49:51,749 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:49:53,750 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:00,621 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:50:05,732 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:50:05,732 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:50:05,732 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:50:05,754 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:50:06,567 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:50:06,568 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:50:07,755 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:09,755 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:11,756 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:13,757 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:21,708 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:50:21,708 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:50:25,776 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:50:25,777 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:50:25,777 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:50:26,761 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:50:27,762 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:29,763 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:30,692 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:50:31,764 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:33,764 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:36,838 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:50:36,838 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:50:45,815 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:50:45,815 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:50:45,815 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:50:46,769 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:50:47,769 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:49,770 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:51,771 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:50:51,971 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:50:51,972 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:50:53,772 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:00,763 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:51:05,871 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:51:05,871 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:51:05,872 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:51:06,776 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:51:07,124 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:51:07,125 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:51:07,777 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:09,778 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:11,778 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:13,779 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:22,256 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:51:22,257 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:51:25,912 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:51:25,913 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:51:25,913 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:51:26,784 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:51:27,784 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:29,785 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:30,835 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:51:31,786 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:33,786 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:37,386 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:51:37,387 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:51:45,992 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:51:45,992 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:51:45,993 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:51:46,791 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:51:47,792 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:49,792 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:51,793 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:51:52,518 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:51:52,518 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:51:53,793 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:00,908 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:52:06,048 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:52:06,049 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:52:06,049 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:52:06,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:52:07,678 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:52:07,679 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:52:08,799 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:10,800 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:12,801 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:14,802 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:22,820 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:52:22,821 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:52:26,094 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:52:26,094 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:52:26,095 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:52:26,806 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:52:28,807 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:30,808 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:30,980 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:52:32,809 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:34,809 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:37,951 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:52:37,952 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:52:46,129 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:52:46,130 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:52:46,130 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:52:46,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:52:48,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:50,815 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:52,816 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:52:53,086 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:52:53,086 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:52:54,817 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:01,053 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:53:06,188 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:53:06,188 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:53:06,189 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:53:06,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:53:08,218 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:53:08,219 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:53:08,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:10,823 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:12,825 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:14,826 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:23,356 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:53:23,356 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:53:26,227 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:53:26,227 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:53:26,227 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:53:26,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:53:28,831 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:30,832 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:31,126 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:53:32,833 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:34,834 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:38,487 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:53:38,487 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:53:46,315 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:53:46,315 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:53:46,316 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:53:46,838 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:53:48,839 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:50,840 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:52,841 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:53:53,622 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:53:53,623 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:53:54,841 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:01,198 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:54:06,354 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:54:06,355 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:54:06,355 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:54:06,846 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:54:08,760 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:54:08,761 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:54:08,847 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:10,848 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:12,849 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:14,849 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:23,899 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:54:23,900 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:54:26,395 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:54:26,395 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:54:26,395 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:54:26,854 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:54:28,855 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:30,855 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:31,275 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:54:32,856 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:34,857 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:39,033 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:54:39,034 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:54:46,430 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:54:46,431 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:54:46,431 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:54:46,861 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:54:48,862 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:50,863 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:52,863 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:54:54,164 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:54:54,164 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:54:54,864 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:01,348 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:55:06,474 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:55:06,475 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:55:06,475 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:55:06,868 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:55:08,869 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:09,295 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:55:09,295 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:55:10,870 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:12,871 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:14,872 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:24,427 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:55:24,428 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:55:26,509 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:55:26,509 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:55:26,510 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:55:26,876 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:55:28,876 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:30,877 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:31,422 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:55:32,878 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:34,879 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:39,559 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:55:39,559 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:55:46,561 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:55:46,561 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:55:46,562 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:55:46,883 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:55:48,883 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:50,884 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:52,885 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:55:54,692 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:55:54,692 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:55:54,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:01,493 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:56:06,599 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:56:06,599 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:56:06,600 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:56:06,890 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:56:08,891 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:09,821 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:56:09,822 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:56:10,892 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:12,893 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:14,893 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:24,959 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:56:24,960 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:56:26,634 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:56:26,634 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:56:26,635 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:56:26,898 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:56:28,899 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:30,899 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:31,566 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:56:32,900 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:34,900 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:40,091 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:56:40,092 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:56:46,674 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:56:46,674 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:56:46,675 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:56:46,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:56:48,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:50,906 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:52,907 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:54,908 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:56:55,226 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:56:55,227 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:57:01,643 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:57:06,709 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:57:06,709 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:57:06,710 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:57:06,912 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:57:08,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:10,356 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:57:10,356 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:57:10,914 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:12,914 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:14,915 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:25,488 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:57:25,488 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:57:26,756 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:57:26,756 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:57:26,757 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:57:26,919 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:57:28,920 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:30,921 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:31,719 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:57:32,922 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:34,923 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:40,623 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:57:40,623 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:57:46,793 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:57:46,793 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:57:46,793 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:57:46,927 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:57:48,928 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:50,929 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:52,930 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:54,931 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:57:55,762 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:57:55,762 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:58:01,793 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:58:06,828 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:58:06,829 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:58:06,829 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:58:06,935 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:58:08,935 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:10,923 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:58:10,924 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:58:10,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:12,937 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:14,938 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:26,051 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:58:26,052 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:58:26,863 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:58:26,863 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:58:26,864 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:58:26,941 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:58:28,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:30,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:31,866 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:58:32,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:34,945 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:41,182 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:58:41,183 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:58:46,898 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:58:46,898 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:58:46,898 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:58:46,949 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:58:48,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:50,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:52,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:54,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:58:56,318 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:58:56,318 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:59:01,941 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:59:06,936 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:59:06,936 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:59:06,937 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:59:06,956 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:59:08,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:10,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:11,446 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:59:11,447 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:59:12,958 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:14,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:26,578 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:59:26,578 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:59:26,977 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:59:26,977 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:59:26,977 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:59:27,964 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:59:28,964 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:30,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:32,016 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 23:59:32,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:34,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:41,712 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:59:41,713 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 23:59:47,017 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 23:59:47,017 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 23:59:47,017 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 23:59:47,971 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 23:59:48,971 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:50,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:52,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:54,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 23:59:56,842 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 23:59:56,842 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:00:02,091 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:00:07,053 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:00:07,054 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:00:07,054 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:00:07,978 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:00:08,979 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:10,979 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:11,972 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:00:11,972 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:00:12,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:14,981 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:27,094 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:00:27,094 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:00:27,095 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:00:27,133 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:00:27,133 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:00:27,985 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:00:28,985 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:30,986 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:32,164 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:00:32,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:34,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:42,266 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:00:42,267 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:00:47,132 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:00:47,133 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:00:47,133 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:00:47,992 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:00:49,993 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:51,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:53,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:55,995 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:00:57,395 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:00:57,396 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:01:02,240 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:01:07,172 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:01:07,173 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:01:07,173 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:01:08,000 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:01:10,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:12,002 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:12,527 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:01:12,528 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:01:14,003 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:16,003 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:27,234 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:01:27,234 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:01:27,234 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:01:27,709 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:01:27,709 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:01:28,008 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:01:30,009 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:32,010 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:32,312 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:01:34,010 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:36,011 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:42,850 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:01:42,851 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:01:47,292 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:01:47,293 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:01:47,293 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:01:48,016 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:01:50,017 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:52,018 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:54,019 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:56,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:01:57,983 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:01:57,984 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:02:02,384 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:02:07,338 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:02:07,339 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:02:07,339 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:02:08,024 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:02:10,025 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:12,026 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:13,215 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:02:13,216 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:02:14,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:16,028 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:27,402 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:02:27,403 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:02:27,403 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:02:28,032 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:02:28,401 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:02:28,402 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:02:30,033 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:32,034 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:32,459 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:02:34,035 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:36,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:43,535 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:02:43,536 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:02:47,453 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:02:47,454 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:02:47,454 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:02:48,040 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:02:50,041 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:52,042 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:54,043 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:56,044 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:02:58,665 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:02:58,665 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:03:02,536 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:03:07,490 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:03:07,490 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:03:07,491 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:03:08,048 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:03:10,049 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:12,050 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:13,798 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:03:13,799 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:03:14,051 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:16,052 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:27,526 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:03:27,527 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:03:27,527 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:03:28,056 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:03:29,001 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:03:29,001 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:03:30,057 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:32,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:32,614 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:03:34,059 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:36,059 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:44,134 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:03:44,134 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:03:47,565 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:03:47,565 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:03:47,566 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:03:48,064 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:03:50,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:52,066 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:54,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:56,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:03:59,265 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:03:59,266 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:04:02,691 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:04:07,601 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:04:07,602 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:04:07,602 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:04:08,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:04:10,073 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:12,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:14,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:14,406 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:04:14,407 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:04:16,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:27,637 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:04:27,637 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:04:27,638 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:04:28,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:04:29,548 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:04:29,548 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:04:30,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:32,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:32,768 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:04:34,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:36,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:44,724 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:04:44,724 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:04:47,675 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:04:47,676 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:04:47,676 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:04:48,086 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:04:50,087 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:52,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:54,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:56,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:04:59,855 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:04:59,856 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:05:02,837 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:05:07,712 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:05:07,712 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:05:07,713 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:05:08,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:05:10,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:12,096 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:14,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:14,988 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:05:14,988 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:05:16,098 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:27,749 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:05:27,749 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:05:27,750 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:05:28,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:05:30,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:30,122 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:05:30,122 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:05:32,103 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:32,910 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:05:34,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:36,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:45,260 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:05:45,261 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:05:47,788 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:05:47,789 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:05:47,789 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:05:48,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:05:50,109 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:52,110 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:54,111 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:05:56,111 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:00,395 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:06:00,396 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:06:02,984 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:06:07,825 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:06:07,825 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:06:07,826 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:06:08,116 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:06:10,117 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:12,117 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:14,118 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:15,525 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:06:15,525 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:06:16,119 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:27,864 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:06:27,865 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:06:27,865 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:06:28,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:06:28,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:30,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:30,660 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:06:30,660 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:06:32,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:33,060 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:06:34,125 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:36,126 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:45,791 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:06:45,791 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:06:47,904 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:06:47,904 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:06:47,904 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:06:48,130 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:06:50,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:52,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:54,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:06:56,133 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:00,923 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:07:00,923 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:07:03,134 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:07:07,955 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:07:07,956 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:07:07,956 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:07:08,139 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:07:10,140 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:12,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:14,142 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:16,058 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:07:16,059 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:07:16,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:27,994 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:07:27,995 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:07:27,995 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:07:28,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:07:30,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:31,191 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:07:31,192 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:07:32,148 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:33,206 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:07:34,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:36,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:46,338 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:07:46,339 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:07:48,030 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:07:48,031 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:07:48,031 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:07:48,153 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:07:51,155 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:53,156 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:55,156 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:07:57,157 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:01,468 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:08:01,469 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:08:03,278 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:08:08,074 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:08:08,074 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:08:08,074 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:08:08,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:08:11,163 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:13,164 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:15,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:16,599 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:08:16,600 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:08:17,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:28,112 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:08:28,113 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:08:28,113 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:08:28,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:08:31,170 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:31,731 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:08:31,731 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:08:33,171 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:33,351 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:08:35,172 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:37,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:46,861 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:08:46,861 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:08:48,151 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:08:48,152 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:08:48,152 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:08:48,177 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:08:51,179 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:53,179 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:55,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:08:57,181 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:01,990 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:09:01,991 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:09:03,422 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:09:08,204 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:09:08,205 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:09:08,205 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:09:09,186 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:09:09,186 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:11,187 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:13,188 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:15,188 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:17,126 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:09:17,126 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:09:17,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:28,242 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:09:28,242 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:09:28,243 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:09:29,193 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:09:29,194 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:31,194 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:32,256 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:09:32,256 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:09:33,195 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:33,497 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:09:35,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:37,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:47,384 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:09:47,384 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:09:48,288 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:09:48,288 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:09:48,289 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:09:49,202 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:09:49,202 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:51,203 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:53,203 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:55,204 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:09:57,205 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:02,515 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:10:02,515 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:10:03,568 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:10:08,330 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:10:08,331 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:10:08,331 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:10:09,210 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:10:09,210 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:11,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:13,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:15,212 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:17,213 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:17,647 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:10:17,647 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:10:28,367 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:10:28,367 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:10:28,368 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:10:29,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:10:29,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:31,219 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:32,776 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:10:32,776 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:10:33,220 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:33,641 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:10:35,221 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:37,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:47,928 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:10:47,928 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:10:48,403 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:10:48,404 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:10:48,404 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:10:49,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:10:49,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:51,228 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:53,229 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:55,230 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:10:57,231 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:03,064 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:11:03,064 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:11:03,715 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:11:08,445 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:11:08,446 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:11:08,446 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:11:09,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:11:09,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:11,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:13,237 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:15,238 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:17,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:18,198 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:11:18,198 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:11:28,484 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:11:28,485 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:11:28,485 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:11:29,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:11:29,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:31,244 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:33,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:33,328 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:11:33,328 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:11:33,786 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:11:35,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:37,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:48,523 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:11:48,523 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:11:48,524 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:11:48,563 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:11:48,564 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:11:49,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:11:49,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:51,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:53,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:55,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:11:57,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:03,699 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:12:03,699 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:12:03,856 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:12:08,565 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:12:08,565 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:12:08,565 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:12:09,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:12:09,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:11,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:13,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:15,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:17,261 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:18,829 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:12:18,830 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:12:28,603 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:12:28,604 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:12:28,604 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:12:29,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:12:29,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:31,266 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:33,267 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:33,933 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:12:33,959 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:12:33,960 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:12:35,267 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:37,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:48,687 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:12:48,688 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:12:48,688 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:12:49,101 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:12:49,101 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:12:49,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:12:49,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:51,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:53,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:55,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:12:57,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:04,007 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:13:04,236 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:13:04,237 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:13:08,730 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:13:08,730 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:13:08,731 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:13:09,279 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:13:09,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:11,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:13,281 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:15,282 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:17,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:19,366 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:13:19,366 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:13:28,767 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:13:28,768 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:13:28,768 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:13:29,288 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:13:29,288 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:32,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:34,080 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:13:34,290 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:34,494 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:13:34,494 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:13:36,291 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:38,292 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:48,807 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:13:48,807 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:13:48,807 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:13:49,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:13:49,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:49,642 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:13:49,642 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:13:50,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:52,297 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:54,298 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:56,299 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:13:58,299 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:04,152 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:14:04,785 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:14:04,785 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:14:08,847 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:14:08,848 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:14:08,848 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:14:09,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:14:09,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:10,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:12,305 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:14,305 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:16,306 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:18,307 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:19,915 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:14:19,915 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:14:28,884 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:14:28,885 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:14:28,885 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:14:29,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:14:29,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:30,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:32,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:34,224 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:14:34,313 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:35,048 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:14:35,048 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:14:36,314 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:38,315 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:40,315 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:42,316 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:44,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:46,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:48,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:50,191 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:14:50,192 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:14:50,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:52,320 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:54,320 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:56,321 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:14:58,322 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:00,322 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:02,323 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:04,298 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:15:04,324 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:05,326 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:15:05,326 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:15:06,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:08,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:10,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:12,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:14,328 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:16,329 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:18,330 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:20,330 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:20,458 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:15:20,459 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:15:22,331 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:24,332 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:26,332 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:28,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:30,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:32,717 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:15:32,718 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:15:32,718 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:15:33,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:15:34,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:34,365 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:15:35,590 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:15:35,591 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:15:36,336 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:38,337 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:40,338 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:50,722 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:15:50,722 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:15:52,762 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:15:52,762 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:15:52,762 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:15:53,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:15:54,343 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:56,344 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:15:58,344 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:00,345 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:04,428 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:16:05,856 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:16:05,857 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:16:12,886 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:16:12,886 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:16:12,887 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:16:13,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:16:14,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:16,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:18,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:20,352 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:20,987 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:16:20,988 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:16:32,906 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:16:32,907 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:16:32,907 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:16:33,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:16:34,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:34,492 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:16:35,358 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:36,121 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:16:36,122 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:16:37,359 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:39,360 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:41,361 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:51,254 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:16:51,254 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:16:52,943 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:16:52,943 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:16:52,943 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:16:53,365 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:16:54,366 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:55,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:57,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:16:59,368 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:01,369 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:04,559 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:17:06,383 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:17:06,383 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:17:12,981 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:17:12,981 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:17:12,982 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:17:13,373 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:17:14,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:15,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:17,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:19,376 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:21,377 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:21,513 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:17:21,514 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:17:33,020 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:17:33,020 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:17:33,021 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:17:33,381 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:17:34,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:34,625 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:17:35,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:36,645 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:17:36,646 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:17:37,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:39,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:41,384 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:51,776 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:17:51,776 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:17:53,058 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:17:53,058 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:17:53,059 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:17:53,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:17:54,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:55,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:57,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:17:59,390 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:01,391 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:04,689 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:18:06,908 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:18:06,909 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:18:13,097 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:18:13,097 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:18:13,098 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:18:13,395 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:18:14,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:15,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:17,397 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:19,398 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:21,399 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:22,037 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:18:22,038 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:18:33,134 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:18:33,134 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:18:33,134 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:18:33,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:18:34,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:34,756 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:18:35,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:37,170 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:18:37,170 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:18:37,406 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:39,407 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:41,408 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:52,301 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:18:52,301 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:18:53,171 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:18:53,171 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:18:53,172 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:18:53,412 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:18:54,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:55,414 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:57,415 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:18:59,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:01,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:04,818 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:19:07,434 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:19:07,435 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:19:13,208 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:19:13,208 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:19:13,208 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:19:13,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:19:14,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:15,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:17,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:19,424 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:21,425 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:22,570 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:19:22,570 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:19:33,248 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:19:33,248 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:19:33,249 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:19:33,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:19:34,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:34,884 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:19:35,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:37,432 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:37,705 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:19:37,705 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:19:39,433 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:41,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:52,841 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:19:52,841 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:19:53,294 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:19:53,294 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:19:53,294 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:19:53,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:19:54,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:55,440 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:57,440 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:19:59,441 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:01,442 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:04,948 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:20:07,971 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:20:07,971 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:20:13,333 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:20:13,333 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:20:13,333 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:20:13,447 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:20:14,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:15,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:17,449 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:19,450 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:21,451 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:23,102 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:20:23,102 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:20:33,370 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:20:33,370 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:20:33,371 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:20:33,456 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:20:34,456 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:35,014 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:20:35,457 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:37,458 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:38,264 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:20:38,264 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:20:39,458 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:41,459 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:53,395 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:20:53,395 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:20:53,523 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:20:53,523 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:20:53,524 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:20:54,464 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:20:55,464 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:57,465 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:20:59,466 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:01,467 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:05,079 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:21:08,540 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:21:08,540 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:21:13,473 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:21:13,473 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:21:13,474 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:21:14,472 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:21:15,472 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:17,473 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:19,473 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:21,474 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:23,672 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:21:23,672 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:21:33,552 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:21:33,553 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:21:33,553 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:21:34,479 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:21:35,149 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:21:35,479 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:37,480 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:38,807 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:21:38,807 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:21:39,481 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:41,482 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:53,590 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:21:53,591 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:21:53,591 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:21:53,942 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:21:53,943 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:21:54,487 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:21:55,488 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:57,488 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:21:59,489 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:01,490 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:05,216 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:22:09,074 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:22:09,074 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:22:13,628 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:22:13,629 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:22:13,629 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:22:14,496 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:22:15,496 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:17,497 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:19,498 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:21,499 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:24,202 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:22:24,202 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:22:33,674 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:22:33,674 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:22:33,675 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:22:34,503 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:22:35,285 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:22:35,504 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:36,504 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:38,505 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:39,375 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:22:39,375 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:22:40,506 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:42,507 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:53,712 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:22:53,713 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:22:53,713 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:22:54,511 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:22:54,568 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:22:54,569 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:22:55,512 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:56,512 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:22:58,513 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:00,514 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:02,515 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:05,354 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:23:09,702 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:23:09,702 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:23:13,766 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:23:13,766 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:23:13,767 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:23:14,520 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:23:15,520 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:16,521 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:18,522 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:20,523 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:22,524 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:24,831 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:23:24,831 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:23:33,806 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:23:33,806 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:23:33,807 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:23:34,529 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:23:35,425 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:23:35,529 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:36,529 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:38,530 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:39,961 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:23:39,961 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:23:40,531 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:42,532 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:53,860 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:23:53,860 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:23:53,861 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:23:54,537 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:23:55,095 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:23:55,096 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:23:55,538 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:56,538 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:23:58,539 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:00,540 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:02,541 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:05,494 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:24:10,225 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:24:10,226 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:24:13,897 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:24:13,898 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:24:13,898 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:24:14,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:24:15,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:16,547 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:18,548 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:20,549 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:22,550 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:25,358 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:24:25,359 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:24:33,935 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:24:33,935 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:24:33,935 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:24:34,554 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:24:35,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:35,563 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:24:36,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:38,556 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:40,489 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:24:40,489 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:24:40,557 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:42,559 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:53,972 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:24:53,973 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:24:53,973 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:24:54,564 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:24:55,564 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:55,627 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:24:55,628 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:24:56,565 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:24:58,566 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:00,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:02,568 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:05,634 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:25:10,758 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:25:10,759 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:25:14,012 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:25:14,012 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:25:14,013 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:25:14,572 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:25:15,572 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:16,573 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:18,574 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:20,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:22,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:25,892 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:25:25,893 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:25:34,051 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:25:34,051 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:25:34,052 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:25:34,579 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:25:35,580 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:35,708 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:25:36,580 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:38,581 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:40,582 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:41,022 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:25:41,023 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:25:42,583 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:54,088 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:25:54,089 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:25:54,089 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:25:54,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:25:55,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:56,198 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:25:56,199 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:25:56,589 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:25:58,590 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:00,591 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:02,592 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:05,779 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:26:11,334 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:26:11,334 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:26:14,128 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:26:14,128 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:26:14,129 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:26:14,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:26:15,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:16,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:18,598 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:20,599 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:22,600 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:26,465 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:26:26,466 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:26:34,173 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:26:34,174 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:26:34,174 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:26:34,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:26:35,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:35,852 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:26:36,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:38,606 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:40,607 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:41,630 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:26:41,630 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:26:42,608 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:54,211 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:26:54,212 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:26:54,212 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:26:54,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:26:55,613 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:56,613 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:26:56,767 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:26:56,768 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:26:58,614 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:00,615 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:02,615 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:05,923 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:27:11,898 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:27:11,898 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:27:14,316 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:27:14,316 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:27:14,316 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:27:14,619 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:27:15,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:16,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:18,621 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:20,622 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:22,623 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:27,030 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:27:27,031 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:27:34,375 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:27:34,375 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:27:34,376 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:27:34,627 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:27:35,627 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:35,997 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:27:36,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:38,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:40,629 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:42,163 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:27:42,163 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:27:42,630 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:54,414 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:27:54,414 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:27:54,415 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:27:54,635 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:27:55,635 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:56,636 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:27:57,323 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:27:57,323 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:27:58,637 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:00,638 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:02,639 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:06,073 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:28:14,005 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:28:14,005 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:28:14,458 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:28:14,458 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:28:14,459 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:28:14,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:28:15,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:16,644 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:18,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:20,646 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:22,647 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:29,139 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:28:29,139 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:28:34,507 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:28:34,507 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:28:34,508 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:28:34,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:28:35,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:36,147 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:28:36,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:38,654 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:40,655 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:42,656 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:44,269 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:28:44,269 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:28:54,550 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:28:54,550 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:28:54,550 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:28:54,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:28:55,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:56,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:58,662 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:28:59,398 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:28:59,398 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:29:00,663 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:02,664 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:06,219 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:29:14,531 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:29:14,531 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:29:14,671 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:29:14,671 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:29:14,672 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:29:15,668 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:29:15,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:16,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:18,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:20,671 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:22,672 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:29,694 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:29:29,694 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:29:34,626 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:29:34,626 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:29:34,627 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:29:34,676 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:29:35,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:36,292 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:29:36,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:38,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:40,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:42,681 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:44,825 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:29:44,825 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:29:54,692 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:29:54,693 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:29:54,693 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:29:55,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:29:55,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:56,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:58,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:29:59,984 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:29:59,984 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:30:00,688 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:02,689 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:06,368 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:30:14,733 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:30:14,733 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:30:14,734 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:30:15,130 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:30:15,130 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:30:15,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:30:16,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:18,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:20,695 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:22,696 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:30,271 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:30:30,271 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:30:34,782 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:30:34,782 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:30:34,783 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:30:35,700 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:30:36,441 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:30:36,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:38,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:40,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:42,703 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:45,401 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:30:45,402 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:30:54,844 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:30:54,845 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:30:54,845 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:30:55,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:30:56,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:30:58,709 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:00,551 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:31:00,551 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:31:00,709 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:02,710 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:06,514 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:31:14,909 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:31:14,909 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:31:14,909 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:31:15,709 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:31:15,710 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:31:15,715 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:31:16,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:18,717 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:20,717 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:22,718 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:30,857 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:31:30,857 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:31:34,974 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:31:34,974 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:31:34,975 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:31:35,723 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:31:36,589 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:31:36,723 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:37,724 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:39,725 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:41,726 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:43,726 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:45,986 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:31:45,986 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:31:55,010 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:31:55,010 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:31:55,011 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:31:55,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:31:56,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:57,732 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:31:59,733 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:01,120 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:32:01,120 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:32:01,734 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:03,734 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:06,663 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:32:15,052 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:32:15,053 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:32:15,053 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:32:15,738 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:32:16,289 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:32:16,289 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:32:16,739 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:17,739 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:19,740 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:21,741 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:23,741 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:31,454 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:32:31,454 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:32:35,089 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:32:35,089 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:32:35,090 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:32:35,745 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:32:36,736 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:32:36,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:37,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:39,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:41,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:43,748 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:46,588 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:32:46,588 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:32:55,132 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:32:55,132 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:32:55,133 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:32:55,753 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:32:56,753 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:57,754 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:32:59,754 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:01,719 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:33:01,719 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:33:01,755 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:03,756 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:06,809 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:33:15,167 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:33:15,168 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:33:15,168 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:33:15,760 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:33:16,760 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:16,851 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:33:16,851 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:33:17,761 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:19,761 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:21,762 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:23,763 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:31,985 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:33:31,985 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:33:35,203 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:33:35,204 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:33:35,204 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:33:35,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:33:36,768 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:36,884 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:33:37,768 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:39,769 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:41,770 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:43,770 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:47,121 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:33:47,121 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:33:55,272 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:33:55,273 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:33:55,273 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:33:55,774 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:33:56,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:57,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:33:59,776 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:01,777 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:02,259 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:34:02,259 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:34:03,777 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:06,956 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:34:15,316 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:34:15,316 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:34:15,317 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:34:15,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:34:16,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:17,394 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:34:17,394 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:34:17,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:19,783 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:21,784 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:23,785 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:32,542 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:34:32,542 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:34:35,362 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:34:35,363 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:34:35,363 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:34:35,788 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:34:36,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:37,025 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:34:37,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:39,790 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:41,791 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:43,792 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:47,676 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:34:47,676 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:34:55,417 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:34:55,418 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:34:55,418 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:34:55,795 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:34:56,796 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:57,796 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:34:59,797 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:01,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:02,809 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:35:02,810 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:35:03,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:07,098 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:35:15,481 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:35:15,481 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:35:15,482 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:35:15,803 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:35:16,803 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:17,804 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:17,948 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:35:17,948 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:35:19,804 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:21,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:23,806 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:33,084 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:35:33,084 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:35:35,518 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:35:35,519 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:35:35,519 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:35:35,810 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:35:36,811 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:37,170 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:35:37,811 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:39,812 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:41,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:43,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:48,214 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:35:48,214 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:35:55,601 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:35:55,601 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:35:55,602 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:35:55,817 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:35:56,818 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:57,818 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:35:59,819 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:01,820 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:03,344 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:36:03,344 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:36:03,821 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:07,242 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:36:15,645 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:36:15,645 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:36:15,646 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:36:15,825 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:36:16,826 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:17,826 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:18,479 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:36:18,480 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:36:19,827 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:21,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:23,829 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:33,614 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:36:33,615 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:36:35,691 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:36:35,691 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:36:35,692 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:36:35,834 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:36:36,835 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:37,316 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:36:37,835 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:39,836 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:41,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:43,838 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:48,753 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:36:48,753 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:36:55,728 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:36:55,729 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:36:55,729 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:36:55,843 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:36:56,843 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:57,844 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:36:59,845 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:01,845 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:03,846 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:03,882 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:37:03,882 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:37:07,389 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:37:15,768 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:37:15,768 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:37:15,768 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:37:15,850 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:37:16,851 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:17,851 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:19,013 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:37:19,014 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:37:19,852 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:21,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:23,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:34,150 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:37:34,150 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:37:35,808 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:37:35,808 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:37:35,809 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:37:35,857 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:37:36,858 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:37,464 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:37:37,858 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:39,859 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:41,860 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:43,860 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:49,283 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:37:49,283 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:37:55,864 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:37:55,865 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:37:55,865 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:37:55,865 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:37:57,866 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:37:59,867 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:01,867 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:03,868 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:04,423 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:38:04,423 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:38:07,538 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:38:15,910 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:38:15,911 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:38:15,911 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:38:16,872 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:38:17,873 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:19,559 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:38:19,559 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:38:19,874 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:21,874 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:23,875 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:34,693 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:38:34,693 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:38:35,960 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:38:35,960 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:38:35,960 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:38:36,879 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:38:37,614 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:38:37,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:39,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:41,881 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:43,882 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:49,823 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:38:49,824 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:38:55,998 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:38:55,998 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:38:55,999 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:38:56,887 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:38:57,887 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:38:59,888 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:01,889 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:03,890 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:04,953 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:39:04,953 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:39:07,692 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:39:16,034 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:39:16,034 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:39:16,035 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:39:16,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:39:17,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:19,895 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:20,087 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:39:20,088 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:39:21,896 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:23,896 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:35,219 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:39:35,219 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:39:36,076 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:39:36,076 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:39:36,076 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:39:36,901 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:39:37,766 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:39:37,901 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:39,902 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:41,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:43,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:50,351 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:39:50,351 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:39:56,122 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:39:56,123 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:39:56,123 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:39:56,907 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:39:57,908 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:39:59,908 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:01,909 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:03,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:05,481 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:40:05,481 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:40:07,839 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:40:16,183 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:40:16,183 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:40:16,184 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:40:16,914 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:40:17,915 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:19,915 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:20,611 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:40:20,612 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:40:21,917 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:24,917 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:35,751 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:40:35,751 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:40:36,228 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:40:36,228 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:40:36,229 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:40:36,921 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:40:37,913 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:40:37,922 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:38,922 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:40,923 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:42,924 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:44,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:50,884 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:40:50,884 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:40:56,265 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:40:56,265 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:40:56,265 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:40:56,929 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:40:57,930 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:40:58,930 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:00,931 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:02,931 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:04,932 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:06,016 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:41:06,016 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:41:07,987 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:41:16,300 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:41:16,301 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:41:16,301 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:41:16,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:41:17,937 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:18,938 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:20,938 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:21,157 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:41:21,157 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:41:22,939 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:24,940 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:36,292 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:41:36,293 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:41:36,423 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:41:36,423 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:41:36,424 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:41:36,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:41:37,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:38,059 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:41:38,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:40,945 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:42,946 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:44,947 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:51,625 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:41:51,626 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:41:56,373 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:41:56,373 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:41:56,374 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:41:56,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:41:57,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:41:58,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:00,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:02,953 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:04,954 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:06,758 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:42:06,759 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:42:08,131 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:42:16,408 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:42:16,408 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:42:16,409 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:42:16,958 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:42:17,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:18,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:20,960 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:21,887 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:42:21,888 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:42:22,961 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:24,961 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:36,444 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:42:36,445 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:42:36,445 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:42:36,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:42:37,017 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:42:37,017 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:42:37,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:38,202 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:42:38,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:40,967 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:42,968 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:44,968 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:52,161 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:42:52,162 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:42:56,480 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:42:56,481 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:42:56,481 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:42:56,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:42:57,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:42:58,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:00,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:02,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:04,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:07,295 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:43:07,296 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:43:08,275 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:43:16,543 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:43:16,543 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:43:16,544 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:43:16,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:43:17,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:18,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:20,981 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:22,425 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:43:22,425 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:43:22,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:24,983 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:36,586 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:43:36,586 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:43:36,586 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:43:36,986 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:43:37,650 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:43:37,650 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:43:37,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:38,350 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:43:38,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:40,988 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:42,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:44,990 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:52,781 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:43:52,781 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:43:56,623 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:43:56,624 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:43:56,624 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:43:56,993 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:43:57,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:43:58,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:00,995 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:02,996 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:04,997 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:07,911 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:44:07,912 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:44:08,421 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:44:16,658 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:44:16,659 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:44:16,659 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:44:17,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:44:18,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:19,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:21,002 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:23,003 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:23,043 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:44:23,043 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:44:25,004 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:36,693 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:44:36,694 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:44:36,694 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:44:37,008 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:44:38,008 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:38,214 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:44:38,215 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:44:38,498 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:44:39,009 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:41,009 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:43,010 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:45,011 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:53,349 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:44:53,349 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:44:56,753 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:44:56,753 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:44:56,754 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:44:57,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:44:58,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:44:59,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:01,016 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:03,017 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:05,018 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:08,481 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:45:08,481 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:45:08,611 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:45:16,791 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:45:16,792 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:45:16,792 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:45:17,022 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:45:18,022 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:19,023 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:21,023 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:23,024 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:23,613 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:45:23,613 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:45:25,025 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:36,849 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:45:36,849 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:45:36,850 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:45:37,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:45:38,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:38,648 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:45:38,776 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:45:38,777 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:45:39,030 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:41,030 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:43,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:45,032 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:53,920 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:45:53,920 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:45:56,885 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:45:56,885 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:45:56,886 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:45:57,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:45:58,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:45:59,037 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:01,038 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:03,038 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:05,039 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:08,722 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:46:09,053 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:46:09,053 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:46:16,921 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:46:16,922 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:46:16,922 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:46:17,044 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:46:18,044 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:19,044 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:21,045 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:23,046 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:24,185 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:46:24,185 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:46:25,047 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:36,959 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:46:36,959 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:46:36,959 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:46:37,051 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:46:38,051 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:38,796 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:46:39,051 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:39,324 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:46:39,324 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:46:41,052 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:43,053 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:45,053 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:46:54,458 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:46:54,458 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:46:56,995 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:46:56,995 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:46:56,995 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:46:57,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:46:59,059 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:01,059 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:03,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:05,061 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:08,870 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:47:09,588 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:47:09,588 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:47:17,040 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:47:17,040 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:47:17,040 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:47:17,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:47:19,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:21,066 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:23,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:24,716 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:47:24,717 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:47:25,068 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:37,079 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:47:37,079 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:47:37,080 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:47:38,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:47:38,942 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:47:39,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:39,855 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:47:39,856 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:47:41,073 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:43,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:45,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:47:54,994 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:47:54,994 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:47:57,117 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:47:57,118 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:47:57,118 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:47:58,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:47:59,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:01,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:03,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:05,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:09,014 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:48:10,126 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:48:10,126 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:48:17,189 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:48:17,189 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:48:17,190 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:48:18,087 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:48:19,087 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:21,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:23,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:25,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:25,258 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:48:25,258 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:48:37,225 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:48:37,225 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:48:37,226 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:48:38,093 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:48:39,088 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:48:39,093 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:40,389 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:48:40,390 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:48:41,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:43,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:45,096 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:48:55,527 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:48:55,527 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:48:57,264 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:48:57,265 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:48:57,265 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:48:58,100 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:48:59,100 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:01,101 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:03,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:05,103 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:09,164 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:49:10,660 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:49:10,660 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:49:17,324 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:49:17,324 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:49:17,324 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:49:18,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:49:19,107 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:21,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:23,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:25,109 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:25,794 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:49:25,794 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:49:37,386 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:49:37,386 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:49:37,387 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:49:38,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:49:39,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:39,235 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:49:40,927 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:49:40,927 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:49:41,115 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:43,115 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:45,116 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:49:56,096 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:49:56,096 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:49:57,433 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:49:57,433 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:49:57,433 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:49:58,120 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:49:59,121 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:00,121 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:02,122 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:04,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:06,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:09,308 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:50:11,228 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:50:11,228 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:50:17,471 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:50:17,472 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:50:17,472 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:50:18,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:50:19,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:20,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:22,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:24,129 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:26,130 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:26,360 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:50:26,360 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:50:37,507 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:50:37,507 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:50:37,508 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:50:38,134 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:50:39,134 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:39,382 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:50:40,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:41,492 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:50:41,493 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:50:42,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:44,136 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:46,137 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:50:56,636 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:50:56,636 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:50:57,562 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:50:57,563 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:50:57,563 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:50:58,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:50:59,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:00,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:02,142 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:04,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:06,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:09,456 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:51:11,767 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:51:11,767 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:51:17,600 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:51:17,601 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:51:17,601 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:51:18,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:51:19,148 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:20,148 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:22,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:24,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:26,150 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:26,910 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:51:26,910 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:51:37,637 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:51:37,637 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:51:37,638 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:51:38,154 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:51:39,154 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:39,530 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:51:40,155 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:42,042 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:51:42,042 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:51:42,155 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:44,156 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:46,157 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:51:57,172 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:51:57,172 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:51:57,671 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:51:57,671 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:51:57,672 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:51:58,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:51:59,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:00,162 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:02,162 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:04,163 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:06,164 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:09,604 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:52:12,305 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:52:12,305 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:52:17,705 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:52:17,705 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:52:17,706 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:52:18,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:52:19,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:20,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:22,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:24,170 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:26,171 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:27,436 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:52:27,436 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:52:37,740 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:52:37,740 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:52:37,740 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:52:38,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:52:39,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:39,676 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:52:40,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:42,176 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:42,566 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:52:42,566 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:52:44,177 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:46,178 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:52:57,698 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:52:57,699 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:52:57,825 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:52:57,825 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:52:57,826 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:52:58,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:53:00,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:02,183 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:04,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:06,185 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:09,748 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:53:12,827 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:53:12,827 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:53:17,809 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:53:17,809 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:53:17,810 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:53:18,188 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:53:20,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:22,190 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:24,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:26,192 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:27,963 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:53:27,964 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:53:37,845 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:53:37,845 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:53:37,846 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:53:38,196 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:53:39,822 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:53:40,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:42,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:43,099 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:53:43,099 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:53:44,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:46,199 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:53:57,888 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:53:57,888 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:53:57,888 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:53:58,203 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:53:58,233 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:53:58,233 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:54:00,204 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:02,205 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:04,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:06,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:09,896 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:54:13,370 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:54:13,371 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:54:17,923 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:54:17,923 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:54:17,924 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:54:18,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:54:20,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:22,212 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:24,213 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:26,214 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:28,503 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:54:28,504 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:54:37,959 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:54:37,959 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:54:37,960 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:54:38,219 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:54:39,967 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:54:40,220 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:42,221 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:43,659 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:54:43,659 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:54:44,221 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:46,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:54:58,004 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:54:58,004 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:54:58,005 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:54:58,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:54:58,799 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:54:58,800 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:55:00,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:02,228 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:04,229 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:06,229 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:10,040 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:55:13,936 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:55:13,936 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:55:18,040 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:55:18,040 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:55:18,041 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:55:18,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:55:20,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:22,235 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:24,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:26,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:29,067 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:55:29,068 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:55:38,089 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:55:38,090 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:55:38,090 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:55:38,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:55:40,113 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:55:40,241 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:42,241 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:44,201 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:55:44,202 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:55:44,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:46,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:48,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:50,244 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:52,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:54,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:56,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:58,247 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:55:59,383 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:55:59,384 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:56:00,247 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:02,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:04,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:06,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:08,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:10,187 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:56:11,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:13,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:14,515 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:56:14,516 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:56:15,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:17,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:19,254 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:21,255 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:23,255 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:25,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:27,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:29,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:29,645 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:56:29,646 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:56:31,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:33,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:35,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:37,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:39,261 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:40,259 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-16 00:56:41,917 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-16 00:56:41,917 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-16 00:56:41,917 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 00:56:42,262 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-16 00:56:44,821 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:56:44,821 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-16 00:56:48,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:52,267 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:56:54,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-16 00:57:00,563 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 00:57:00,642 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status