diff --git "a/wandb/run-20210715_152938-8qznp93p/logs/debug-internal.log" "b/wandb/run-20210715_152938-8qznp93p/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20210715_152938-8qznp93p/logs/debug-internal.log" @@ -0,0 +1,2793 @@ +2021-07-15 15:29:39,144 INFO MainThread:708348 [internal.py:wandb_internal():88] W&B internal server running at pid: 708348, started at: 2021-07-15 15:29:39.144511 +2021-07-15 15:29:39,147 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: check_version +2021-07-15 15:29:39,147 INFO WriterThread:708348 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/run-8qznp93p.wandb +2021-07-15 15:29:39,148 DEBUG SenderThread:708348 [sender.py:send():179] send: header +2021-07-15 15:29:39,148 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: check_version +2021-07-15 15:29:39,185 DEBUG SenderThread:708348 [sender.py:send():179] send: run +2021-07-15 15:29:39,355 INFO SenderThread:708348 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files +2021-07-15 15:29:39,355 INFO SenderThread:708348 [sender.py:_start_run_threads():716] run started: 8qznp93p with start time 1626362978 +2021-07-15 15:29:39,355 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:29:39,355 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:29:39,355 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: run_start +2021-07-15 15:29:40,357 INFO Thread-8 :708348 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:29:40,447 DEBUG HandlerThread:708348 [meta.py:__init__():39] meta init +2021-07-15 15:29:40,447 DEBUG HandlerThread:708348 [meta.py:__init__():53] meta init done +2021-07-15 15:29:40,447 DEBUG HandlerThread:708348 [meta.py:probe():210] probe +2021-07-15 15:29:40,449 DEBUG HandlerThread:708348 [meta.py:_setup_git():200] setup git +2021-07-15 15:29:40,481 DEBUG HandlerThread:708348 [meta.py:_setup_git():207] setup git done +2021-07-15 15:29:40,482 DEBUG HandlerThread:708348 [meta.py:_save_pip():57] save pip +2021-07-15 15:29:40,482 DEBUG HandlerThread:708348 [meta.py:_save_pip():71] save pip done +2021-07-15 15:29:40,482 DEBUG HandlerThread:708348 [meta.py:probe():252] probe done +2021-07-15 15:29:40,485 DEBUG SenderThread:708348 [sender.py:send():179] send: files +2021-07-15 15:29:40,485 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-15 15:29:40,491 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:29:40,492 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:29:40,623 DEBUG SenderThread:708348 [sender.py:send():179] send: config +2021-07-15 15:29:40,623 DEBUG SenderThread:708348 [sender.py:send():179] send: config +2021-07-15 15:29:40,624 DEBUG SenderThread:708348 [sender.py:send():179] send: config +2021-07-15 15:29:40,935 INFO Thread-11 :708348 [upload_job.py:push():137] Uploaded file /tmp/tmpl0_rd8g4wandb/1jhnz0nm-wandb-metadata.json +2021-07-15 15:29:41,356 INFO Thread-8 :708348 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/requirements.txt +2021-07-15 15:29:41,356 INFO Thread-8 :708348 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-metadata.json +2021-07-15 15:29:41,356 INFO Thread-8 :708348 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:29:55,718 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:29:55,719 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:29:56,362 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:29:58,362 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:30:08,537 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:30:10,368 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/config.yaml +2021-07-15 15:30:10,861 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:30:10,861 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:30:25,992 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:30:25,992 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:30:38,609 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:30:41,132 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:30:41,132 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:30:56,272 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:30:56,272 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:31:08,674 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:31:11,407 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:31:11,408 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:31:26,550 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:31:26,551 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:31:38,740 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:31:41,983 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:31:41,983 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:31:57,113 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:31:57,114 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:32:08,807 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:32:12,246 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:32:12,246 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:32:27,383 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:32:27,384 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:32:38,873 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:32:42,519 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:32:42,519 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:32:57,651 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:32:57,652 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:33:08,941 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:33:12,784 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:33:12,784 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:33:27,921 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:33:27,921 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:33:39,012 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:33:43,059 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:33:43,060 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:33:58,193 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:33:58,194 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:34:09,086 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:34:13,326 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:34:13,327 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:34:28,459 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:34:28,459 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:34:39,160 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:34:43,592 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:34:43,592 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:34:58,723 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:34:58,723 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:35:09,235 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:35:13,859 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:35:13,859 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:35:24,483 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:35:29,011 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:35:29,012 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:35:39,307 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:35:44,170 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:35:44,170 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:35:59,323 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:35:59,323 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:36:09,385 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:36:14,458 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:36:14,458 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:36:29,600 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:36:29,601 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:36:39,464 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:36:44,734 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:36:44,734 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:36:59,866 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:36:59,866 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:37:09,541 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:37:15,002 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:37:15,003 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:37:29,534 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:37:30,147 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:37:30,148 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:37:39,618 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:37:45,296 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:37:45,296 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:38:00,446 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:38:00,447 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:38:09,696 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:38:15,586 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:38:15,587 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:38:30,719 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:38:30,719 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:38:39,778 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:38:45,850 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:38:45,851 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:39:00,984 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:39:00,985 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:39:09,859 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:39:16,117 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:39:16,117 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:39:31,259 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:39:31,259 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:39:39,943 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:39:41,585 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:39:43,586 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:39:45,587 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:39:46,394 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:39:46,394 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:39:47,588 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:39:58,903 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:39:58,904 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:39:58,904 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:39:59,592 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:40:01,528 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:40:01,528 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:40:01,593 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:03,593 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:05,594 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:07,595 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:10,013 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:40:16,661 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:40:16,661 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:40:18,940 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:40:18,940 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:40:18,940 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:40:19,599 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:40:21,601 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:23,601 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:25,602 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:27,603 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:31,791 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:40:31,792 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:40:38,976 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:40:38,976 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:40:38,976 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:40:39,607 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:40:40,087 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:40:41,608 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:43,609 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:45,610 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:46,953 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:40:46,954 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:40:47,610 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:40:59,014 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:40:59,015 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:40:59,015 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:40:59,615 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:41:01,615 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:02,083 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:41:02,083 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:41:03,616 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:05,617 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:07,618 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:10,157 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:41:17,215 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:41:17,215 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:41:19,053 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:41:19,054 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:41:19,054 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:41:19,622 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:41:19,622 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:21,623 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:23,624 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:25,624 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:27,625 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:32,348 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:41:32,349 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:41:39,095 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:41:39,095 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:41:39,095 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:41:39,630 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:41:39,630 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:40,229 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:41:41,631 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:43,631 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:45,632 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:47,480 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:41:47,480 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:41:47,633 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:41:59,131 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:41:59,131 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:41:59,132 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:41:59,637 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:41:59,637 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:01,637 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:02,612 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:42:02,613 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:42:04,639 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:06,640 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:08,640 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:10,304 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:42:17,746 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:42:17,746 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:42:19,166 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:42:19,167 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:42:19,167 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:42:19,645 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:42:20,645 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:21,645 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:22,646 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:24,646 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:26,647 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:28,648 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:32,879 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:42:32,880 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:42:39,205 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:42:39,205 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:42:39,206 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:42:39,652 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:42:40,377 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:42:40,653 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:41,653 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:42,654 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:44,654 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:46,655 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:48,010 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:42:48,010 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:42:48,656 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:42:59,243 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:42:59,243 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:42:59,244 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:42:59,660 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:43:00,660 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:01,661 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:02,661 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:03,145 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:43:03,146 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:43:04,662 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:06,663 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:08,664 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:10,452 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:43:18,278 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:43:18,278 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:43:19,280 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:43:19,281 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:43:19,281 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:43:19,668 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:43:20,668 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:21,668 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:22,669 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:24,670 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:26,671 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:28,671 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:33,412 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:43:33,412 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:43:39,325 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:43:39,326 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:43:39,326 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:43:39,676 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:43:40,525 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:43:40,676 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:41,677 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:42,677 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:44,678 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:46,679 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:48,544 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:43:48,544 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:43:48,680 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:43:59,362 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:43:59,363 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:43:59,363 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:43:59,685 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:44:00,685 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:01,686 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:02,686 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:03,682 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:44:03,682 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:44:04,687 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:06,688 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:08,689 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:10,603 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:44:18,813 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:44:18,813 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:44:19,404 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:44:19,405 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:44:19,405 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:44:19,693 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:44:19,693 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:20,693 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:22,694 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:24,695 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:26,696 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:28,697 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:33,944 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:44:33,944 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:44:39,442 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:44:39,443 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:44:39,443 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:44:39,700 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:44:39,701 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:40,677 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:44:40,701 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:42,702 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:44,702 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:46,703 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:48,704 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:44:49,075 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:44:49,075 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:44:59,481 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:44:59,481 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:44:59,482 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:44:59,709 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:44:59,709 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:00,709 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:02,710 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:04,205 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:45:04,205 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:45:04,711 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:06,711 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:08,712 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:10,753 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:45:19,343 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:45:19,344 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:45:19,522 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:45:19,522 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:45:19,523 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:45:19,717 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:45:20,717 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:22,718 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:24,719 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:26,720 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:28,721 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:34,478 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:45:34,478 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:45:39,561 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:45:39,561 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:45:39,562 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:45:39,725 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:45:40,725 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:40,830 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:45:42,726 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:44,727 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:46,728 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:48,729 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:45:49,608 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:45:49,609 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:45:59,617 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:45:59,617 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:45:59,618 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:45:59,734 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:46:00,734 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:02,735 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:04,736 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:04,738 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:46:04,738 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:46:06,736 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:08,737 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:10,903 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:46:19,659 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:46:19,659 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:46:19,660 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:46:19,741 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:46:19,953 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:46:19,953 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:46:20,742 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:22,742 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:24,743 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:26,744 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:28,745 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:35,091 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:46:35,092 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:46:39,694 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:46:39,694 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:46:39,695 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:46:39,749 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:46:40,749 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:40,981 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:46:42,750 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:44,751 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:46,752 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:48,753 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:46:50,232 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:46:50,232 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:46:59,733 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:46:59,733 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:46:59,734 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:46:59,756 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:47:00,757 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:02,758 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:04,759 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:05,362 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:47:05,363 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:47:06,760 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:08,761 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:11,059 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:47:19,770 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:47:19,770 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:47:19,771 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:47:20,521 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:47:20,521 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:47:20,765 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:47:20,765 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:22,766 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:24,767 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:26,768 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:28,768 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:35,653 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:47:35,653 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:47:39,806 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:47:39,806 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:47:39,807 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:47:40,773 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:47:40,773 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:41,137 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:47:42,774 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:44,775 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:46,776 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:48,777 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:47:50,784 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:47:50,784 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:47:59,842 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:47:59,843 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:47:59,843 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:48:00,781 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:48:00,781 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:02,782 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:04,783 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:05,930 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:48:05,930 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:48:06,784 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:08,785 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:11,214 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:48:19,877 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:48:19,877 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:48:19,878 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:48:20,789 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:48:20,789 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:21,064 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:48:21,064 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:48:22,790 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:24,791 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:26,791 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:28,792 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:36,254 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:48:36,255 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:48:39,914 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:48:39,915 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:48:39,915 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:48:40,797 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:48:40,797 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:41,291 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:48:42,797 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:44,798 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:46,799 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:48,800 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:48:51,386 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:48:51,386 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:48:59,952 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:48:59,953 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:48:59,953 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:49:00,805 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:49:00,805 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:02,806 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:04,807 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:06,520 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:49:06,520 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:49:06,808 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:08,808 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:11,370 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:49:19,991 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:49:19,992 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:49:19,992 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:49:20,813 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:49:20,813 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:22,813 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:24,512 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:49:24,512 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:49:24,814 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:26,815 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:28,816 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:39,645 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:49:39,646 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:49:40,033 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:49:40,034 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:49:40,034 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:49:40,820 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:49:41,447 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:49:41,821 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:42,821 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:43,821 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:45,822 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:47,823 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:49,824 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:49:54,776 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:49:54,776 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:50:00,080 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:50:00,081 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:50:00,081 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:50:00,828 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:50:01,829 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:02,829 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:03,830 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:05,831 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:07,832 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:09,832 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:09,908 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:50:09,909 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:50:11,525 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:50:20,127 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:50:20,127 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:50:20,128 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:50:20,837 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:50:20,837 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:21,837 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:23,838 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:25,040 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:50:25,040 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:50:25,839 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:27,840 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:29,841 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:40,168 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:50:40,169 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:50:40,169 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:50:40,208 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:50:40,208 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:50:40,845 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:50:40,845 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:41,603 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:50:41,845 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:43,846 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:45,847 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:47,848 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:49,849 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:50:55,346 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:50:55,346 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:51:00,207 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:51:00,208 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:51:00,208 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:51:00,853 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:51:00,853 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:01,853 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:03,854 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:05,855 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:07,856 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:09,857 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:10,475 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:51:10,476 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:51:11,678 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:51:20,243 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:51:20,244 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:51:20,244 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:51:20,861 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:51:20,861 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:21,861 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:23,862 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:25,607 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:51:25,607 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:51:25,863 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:27,864 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:29,865 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:40,286 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:51:40,286 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:51:40,287 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:51:40,745 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:51:40,745 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:51:40,868 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:51:40,869 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:41,759 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:51:41,869 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:43,870 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:45,871 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:47,871 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:49,872 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:51:55,910 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:51:55,910 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:52:00,339 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:52:00,339 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:52:00,339 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:52:00,876 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:52:00,876 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:01,876 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:03,877 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:05,878 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:07,879 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:09,880 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:11,042 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:52:11,042 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:52:11,836 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:52:20,374 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:52:20,374 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:52:20,375 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:52:20,884 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:52:20,884 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:21,884 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:23,885 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:25,886 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:26,175 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:52:26,176 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:52:27,886 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:29,887 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:40,415 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:52:40,415 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:52:40,416 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:52:40,891 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:52:41,357 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:52:41,357 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:52:41,891 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:41,914 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:52:43,892 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:45,893 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:47,894 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:49,895 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:52:56,500 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:52:56,500 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:53:00,461 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:53:00,461 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:53:00,462 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:53:00,899 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:53:01,899 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:03,900 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:05,901 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:07,902 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:09,903 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:11,631 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:53:11,632 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:53:11,992 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:53:20,498 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:53:20,498 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:53:20,499 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:53:20,907 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:53:21,907 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:23,908 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:25,909 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:26,767 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:53:26,767 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:53:27,910 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:29,911 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:40,538 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:53:40,538 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:53:40,539 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:53:40,914 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:53:41,915 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:41,918 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:53:41,919 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:53:42,069 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:53:43,916 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:45,917 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:47,918 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:49,918 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:53:57,058 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:53:57,059 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:54:00,576 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:54:00,576 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:54:00,577 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:54:00,922 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:54:01,923 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:03,924 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:05,925 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:07,925 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:09,926 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:12,148 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:54:12,201 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:54:12,202 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:54:20,612 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:54:20,613 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:54:20,613 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:54:20,930 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:54:21,931 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:23,932 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:25,933 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:27,333 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:54:27,334 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:54:27,934 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:29,934 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:40,651 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:54:40,651 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:54:40,652 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:54:40,939 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:54:41,939 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:42,226 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:54:42,527 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:54:42,528 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:54:43,940 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:45,941 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:47,942 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:49,943 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:54:57,673 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:54:57,674 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:55:00,688 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:55:00,689 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:55:00,689 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:55:00,947 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:55:01,948 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:03,948 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:05,949 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:07,950 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:09,951 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:12,304 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:55:12,816 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:55:12,816 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:55:20,733 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:55:20,733 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:55:20,734 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:55:20,955 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:55:21,956 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:23,957 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:25,958 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:27,948 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:55:27,948 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:55:27,958 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:29,960 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:40,776 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:55:40,777 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:55:40,777 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:55:40,964 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:55:41,964 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:42,382 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:55:43,120 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:55:43,121 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:55:43,965 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:45,966 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:47,967 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:49,968 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:55:58,281 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:55:58,281 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:56:00,823 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:56:00,824 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:56:00,824 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:56:00,972 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:56:01,973 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:03,974 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:05,974 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:07,975 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:09,976 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:12,459 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:56:13,414 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:56:13,414 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:56:20,862 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:56:20,862 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:56:20,863 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:56:20,981 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:56:21,981 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:23,982 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:25,983 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:27,984 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:28,552 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:56:28,552 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:56:29,985 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:40,899 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:56:40,900 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:56:40,900 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:56:40,989 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:56:41,990 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:42,535 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:56:43,683 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:56:43,683 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:56:43,990 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:45,991 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:47,992 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:49,993 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:56:59,018 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:56:59,018 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:57:00,938 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:57:00,939 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:57:00,939 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:57:00,998 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:57:01,998 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:03,999 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:06,000 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:08,001 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:10,002 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:12,618 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:57:14,152 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:57:14,152 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:57:20,977 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:57:20,977 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:57:20,978 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:57:21,006 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:57:22,007 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:24,008 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:26,008 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:28,009 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:29,284 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:57:29,284 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:57:30,010 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:41,016 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:57:41,017 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:57:41,017 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:57:42,015 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:57:42,015 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:42,696 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:57:44,016 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:44,413 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:57:44,414 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:57:46,016 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:48,017 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:50,018 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:57:59,546 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:57:59,547 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:58:01,051 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:58:01,052 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:58:01,052 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:58:02,023 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:58:02,024 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:04,024 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:06,025 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:08,026 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:10,027 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:12,774 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:58:14,680 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:58:14,681 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:58:21,093 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:58:21,093 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:58:21,093 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:58:22,032 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:58:22,032 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:24,033 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:26,033 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:28,034 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:29,813 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:58:29,814 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:58:30,035 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:41,130 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:58:41,130 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:58:41,131 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:58:42,040 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:58:42,040 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:42,852 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:58:44,041 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:44,948 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:58:44,948 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:58:47,042 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:49,043 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:58:51,044 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:00,084 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:59:00,084 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:59:01,169 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:59:01,169 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:59:01,170 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:59:02,049 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:59:02,049 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:03,049 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:05,050 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:07,051 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:09,052 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:11,053 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:12,929 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:59:15,216 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:59:15,216 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:59:21,213 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:59:21,214 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:59:21,214 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:59:22,058 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:59:22,058 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:23,058 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:25,059 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:27,059 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:29,061 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:30,357 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:59:30,358 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 15:59:31,062 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:41,250 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 15:59:41,251 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 15:59:41,251 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 15:59:42,066 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 15:59:42,066 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:43,007 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 15:59:43,067 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 15:59:45,500 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 15:59:45,500 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:00:00,658 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:00:00,658 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:00:03,074 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:05,075 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:07,075 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:09,076 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:11,077 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:13,077 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:13,083 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:00:15,078 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:17,079 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:17,262 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:00:17,263 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:00:19,080 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:21,080 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:23,081 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:25,082 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:27,083 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:29,084 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:31,085 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:32,398 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:00:32,399 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:00:33,085 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:35,086 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:37,087 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:39,088 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:41,089 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:43,090 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:43,156 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:00:46,091 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:47,569 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:00:47,570 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:00:48,092 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:50,093 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:52,093 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:54,094 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:56,095 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:00:58,096 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:00,097 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:02,098 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:02,700 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:01:02,700 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:01:03,595 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:01:03,595 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:01:03,596 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:01:04,099 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:01:06,100 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:08,101 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:10,102 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:12,103 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:13,236 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:01:17,829 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:01:17,830 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:01:23,643 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:01:23,643 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:01:23,644 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:01:24,108 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:01:25,108 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:26,109 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:28,109 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:30,110 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:32,111 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:32,963 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:01:32,964 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:01:43,313 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:01:43,685 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:01:43,685 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:01:43,686 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:01:44,116 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:01:45,117 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:46,117 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:48,094 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:01:48,095 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:01:48,118 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:50,119 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:01:52,120 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:03,228 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:02:03,229 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:02:03,722 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:02:03,723 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:02:03,723 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:02:04,124 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:02:05,125 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:06,125 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:08,126 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:10,127 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:12,128 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:13,391 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:02:18,359 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:02:18,359 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:02:23,762 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:02:23,762 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:02:23,762 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:02:24,133 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:02:25,133 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:26,134 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:28,134 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:30,135 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:32,136 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:33,493 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:02:33,493 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:02:43,469 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:02:43,798 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:02:43,798 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:02:43,799 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:02:44,141 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:02:45,142 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:46,142 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:48,143 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:48,623 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:02:48,623 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:02:50,144 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:02:52,145 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:03,756 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:03:03,757 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:03:03,893 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:03:03,893 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:03:03,894 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:03:04,150 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:03:05,150 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:06,150 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:08,151 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:10,152 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:12,153 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:13,547 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:03:18,899 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:03:18,899 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:03:23,881 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:03:23,882 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:03:23,882 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:03:24,158 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:03:25,158 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:26,159 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:28,160 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:30,161 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:32,161 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:34,029 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:03:34,029 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:03:43,625 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:03:43,916 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:03:43,916 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:03:43,917 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:03:44,166 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:03:45,167 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:46,167 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:48,168 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:49,204 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:03:49,204 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:03:50,169 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:03:52,170 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:03,952 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:04:03,952 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:04:03,953 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:04:04,174 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:04:04,342 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:04:04,342 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:04:05,175 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:06,175 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:08,176 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:10,177 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:12,178 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:13,703 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:04:19,506 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:04:19,507 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:04:23,991 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:04:23,991 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:04:23,992 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:04:24,182 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:04:25,183 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:26,183 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:28,184 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:30,185 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:32,185 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:34,638 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:04:34,639 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:04:43,779 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:04:44,031 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:04:44,031 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:04:44,031 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:04:44,190 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:04:45,190 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:46,191 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:48,192 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:49,772 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:04:49,772 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:04:50,193 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:04:52,193 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:04,074 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:05:04,075 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:05:04,075 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:05:04,198 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:05:04,903 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:05:04,903 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:05:05,199 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:06,199 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:08,200 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:10,201 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:12,202 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:13,857 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:05:20,041 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:05:20,042 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:05:24,127 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:05:24,128 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:05:24,128 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:05:24,207 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:05:25,207 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:26,208 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:28,208 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:30,209 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:32,210 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:35,194 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:05:35,194 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:05:43,934 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:05:44,170 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:05:44,171 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:05:44,171 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:05:44,215 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:05:45,215 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:46,216 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:48,216 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:50,217 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:05:50,325 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:05:50,326 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:05:52,218 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:04,208 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:06:04,208 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:06:04,209 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:06:04,223 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:06:05,224 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:05,481 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:06:05,482 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:06:06,224 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:08,225 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:10,226 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:12,226 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:14,013 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:06:20,622 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:06:20,622 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:06:24,248 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:06:24,248 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:06:24,249 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:06:25,232 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:06:25,232 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:26,232 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:28,233 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:30,234 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:32,235 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:35,827 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:06:35,827 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:06:44,090 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:06:44,293 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:06:44,293 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:06:44,293 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:06:45,240 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:06:45,240 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:46,240 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:48,241 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:50,242 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:06:51,017 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:06:51,018 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:06:52,242 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:04,332 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:07:04,332 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:07:04,332 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:07:05,248 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:07:05,248 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:06,183 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:07:06,184 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:07:06,248 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:08,249 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:10,250 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:12,250 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:14,167 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:07:21,371 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:07:21,371 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:07:24,374 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:07:24,375 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:07:24,375 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:07:25,255 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:07:25,255 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:26,256 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:28,256 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:30,257 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:32,258 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:36,504 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:07:36,505 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:07:44,245 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:07:44,411 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:07:44,411 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:07:44,412 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:07:45,263 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:07:45,263 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:46,264 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:48,264 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:50,265 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:07:51,635 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:07:51,636 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:07:52,266 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:04,471 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:08:04,472 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:08:04,472 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:08:05,271 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:08:05,271 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:06,271 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:06,783 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:08:06,783 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:08:08,272 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:10,273 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:12,274 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:14,322 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:08:21,915 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:08:21,916 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:08:24,507 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:08:24,507 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:08:24,508 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:08:25,278 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:08:25,279 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:26,279 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:28,280 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:30,280 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:33,282 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:37,046 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:08:37,047 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:08:44,400 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:08:44,551 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:08:44,551 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:08:44,551 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:08:45,286 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:08:46,287 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:47,287 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:49,288 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:51,289 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:08:52,179 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:08:52,179 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:08:53,290 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:04,594 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:09:04,594 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:09:04,595 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:09:05,294 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:09:06,294 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:07,295 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:07,311 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:09:07,312 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:09:09,296 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:11,296 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:13,297 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:14,477 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:09:22,442 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:09:22,443 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:09:24,634 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:09:24,635 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:09:24,635 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:09:25,302 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:09:26,303 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:27,303 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:29,304 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:31,304 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:33,305 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:37,581 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:09:37,582 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:09:44,556 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:09:44,673 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:09:44,673 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:09:44,674 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:09:45,310 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:09:46,311 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:47,311 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:49,312 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:51,313 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:09:52,713 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:09:52,713 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:09:53,314 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:04,711 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:10:04,711 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:10:04,712 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:10:05,319 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:10:06,319 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:07,320 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:07,845 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:10:07,846 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:10:09,320 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:11,321 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:13,322 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:14,632 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:10:23,008 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:10:23,009 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:10:24,748 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:10:24,748 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:10:24,749 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:10:25,328 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:10:26,328 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:27,328 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:29,329 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:31,330 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:33,331 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:38,138 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:10:38,138 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:10:44,699 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:10:44,785 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:10:44,786 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:10:44,786 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:10:45,336 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:10:46,337 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:47,337 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:49,338 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:51,339 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:10:53,271 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:10:53,271 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:10:53,340 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:04,826 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:11:04,827 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:11:04,827 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:11:05,344 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:11:06,345 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:07,345 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:08,403 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:11:08,403 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:11:09,346 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:11,347 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:13,348 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:14,771 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:11:23,543 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:11:23,543 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:11:24,869 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:11:24,870 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:11:24,870 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:11:25,353 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:11:26,353 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:27,354 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:29,355 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:31,355 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:33,356 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:38,677 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:11:38,677 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:11:44,847 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:11:44,908 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:11:44,908 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:11:44,909 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:11:45,362 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:11:46,362 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:47,363 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:49,364 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:51,364 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:53,365 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:11:53,807 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:11:53,807 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:12:04,952 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:12:04,952 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:12:04,953 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:12:05,370 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:12:06,370 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:07,371 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:08,936 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:12:08,936 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:12:09,372 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:11,373 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:13,373 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:14,925 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:12:24,077 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:12:24,077 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:12:24,988 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:12:24,989 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:12:24,989 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:12:25,378 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:12:26,378 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:27,379 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:29,380 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:31,380 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:33,381 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:39,210 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:12:39,210 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:12:45,000 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:12:45,031 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:12:45,031 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:12:45,031 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:12:45,386 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:12:46,386 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:47,387 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:49,388 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:51,388 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:53,389 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:12:54,343 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:12:54,344 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:13:05,073 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:13:05,074 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:13:05,074 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:13:05,393 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:13:06,394 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:07,394 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:09,395 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:09,477 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:13:09,478 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:13:11,396 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:13,397 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:15,076 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:13:24,625 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:13:24,625 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:13:25,111 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:13:25,111 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:13:25,112 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:13:25,402 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:13:26,402 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:27,402 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:29,403 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:31,404 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:33,405 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:39,756 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:13:39,756 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:13:45,145 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:13:45,146 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:13:45,148 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:13:45,153 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:13:45,410 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:13:46,410 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:47,411 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:49,412 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:51,412 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:53,413 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:13:54,889 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:13:54,889 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:14:05,180 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:14:05,180 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:14:05,181 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:14:05,418 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:14:06,418 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:07,419 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:09,420 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:10,020 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:14:10,021 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:14:11,420 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:13,421 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:15,231 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:14:25,153 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:14:25,153 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:14:25,283 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:14:25,283 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:14:25,284 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:14:25,426 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:14:26,427 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:27,427 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:29,428 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:31,429 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:33,429 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:40,290 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:14:40,290 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:14:45,258 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:14:45,258 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:14:45,260 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:14:45,309 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:14:45,434 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:14:46,435 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:47,435 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:49,436 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:51,437 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:53,437 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:14:55,428 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:14:55,428 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:15:05,298 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:15:05,298 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:15:05,298 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:15:05,442 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:15:06,442 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:07,442 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:09,443 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:10,589 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:15:10,589 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:15:11,444 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:13,445 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:15,386 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:15:25,336 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:15:25,336 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:15:25,337 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:15:25,449 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:15:25,733 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:15:25,733 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:15:26,449 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:27,450 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:29,451 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:31,451 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:33,452 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:40,866 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:15:40,866 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:15:45,374 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:15:45,375 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:15:45,375 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:15:45,457 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:15:45,464 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:15:46,457 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:47,457 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:49,458 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:51,459 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:53,460 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:15:56,009 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:15:56,010 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:16:05,411 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:16:05,411 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:16:05,412 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:16:05,464 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:16:07,465 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:09,466 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:11,148 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:16:11,148 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:16:11,467 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:13,467 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:15,541 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:16:25,453 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:16:25,453 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:16:25,453 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:16:25,472 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:16:27,336 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:16:27,336 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:16:27,473 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:29,474 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:31,475 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:33,475 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:42,481 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:16:42,481 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:16:45,490 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:16:45,491 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:16:45,491 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:16:45,620 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:16:46,481 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:16:47,481 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:49,482 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:51,483 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:53,483 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:16:57,611 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:16:57,611 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:17:05,529 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:17:05,529 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:17:05,529 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:17:06,488 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:17:07,489 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:09,490 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:11,490 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:12,742 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:17:12,742 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:17:13,491 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:15,697 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:17:25,565 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:17:25,566 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:17:25,566 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:17:26,496 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:17:27,496 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:27,896 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:17:27,897 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:17:29,497 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:31,498 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:33,499 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:43,069 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:17:43,069 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:17:45,602 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:17:45,602 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:17:45,603 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:17:45,774 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:17:46,504 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:17:47,504 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:49,505 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:51,506 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:53,507 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:17:58,201 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:17:58,201 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:18:05,649 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:18:05,649 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:18:05,650 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:18:06,512 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:18:07,512 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:08,512 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:10,513 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:12,514 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:13,339 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:18:13,340 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:18:14,515 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:15,853 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:18:25,685 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:18:25,685 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:18:25,686 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:18:26,520 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:18:27,520 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:28,472 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:18:28,473 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:18:28,520 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:30,521 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:32,522 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:34,523 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:43,604 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:18:43,604 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:18:45,721 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:18:45,722 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:18:45,722 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:18:45,930 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:18:46,528 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:18:47,528 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:48,528 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:50,529 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:52,530 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:54,531 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:18:58,818 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:18:58,818 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:19:05,766 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:19:05,766 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:19:05,767 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:19:06,535 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:19:07,536 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:08,536 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:10,537 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:12,538 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:13,957 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:19:13,957 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:19:14,539 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:16,115 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:19:25,807 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:19:25,807 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:19:25,808 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:19:26,543 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:19:27,544 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:28,544 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:29,224 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:19:29,224 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:19:30,545 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:32,546 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:34,547 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:44,357 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:19:44,357 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:19:45,846 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:19:45,846 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:19:45,847 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:19:46,191 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:19:46,551 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:19:47,551 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:48,552 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:50,553 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:52,553 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:54,554 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:19:59,486 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:19:59,486 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:20:05,893 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:20:05,893 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:20:05,893 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:20:06,559 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:20:07,559 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:08,560 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:10,560 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:12,561 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:14,562 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:14,618 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:20:14,618 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:20:16,269 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:20:25,932 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:20:25,932 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:20:25,933 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:20:26,567 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:20:27,568 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:28,568 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:29,758 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:20:29,759 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:20:30,569 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:32,570 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:34,571 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:44,893 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:20:44,893 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:20:45,968 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:20:45,968 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:20:45,968 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:20:46,346 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:20:46,575 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:20:47,576 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:48,576 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:50,577 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:52,578 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:20:54,579 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:00,034 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:21:00,035 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:21:06,007 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:21:06,007 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:21:06,008 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:21:06,584 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:21:07,584 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:08,584 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:10,585 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:12,586 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:14,587 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:15,190 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:21:15,191 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:21:16,424 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:21:16,588 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:18,588 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:20,589 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:22,590 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:24,591 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:26,592 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:28,592 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:30,324 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:21:30,325 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:21:30,593 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:32,594 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:34,595 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:36,596 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:38,596 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:40,597 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:42,598 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:44,599 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:45,496 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:21:45,497 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:21:46,502 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:21:46,600 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:48,600 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:50,601 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:52,602 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:54,603 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:56,603 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:21:58,604 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:00,605 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:00,684 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:22:00,684 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:22:02,606 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:04,607 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:06,607 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:09,864 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:22:09,864 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:22:09,865 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:22:10,609 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:22:10,609 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:12,610 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:14,611 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:15,816 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:22:15,816 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:22:16,576 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:22:16,611 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:18,612 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:29,913 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:22:29,913 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:22:29,914 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:22:30,617 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:22:30,617 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:30,976 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:22:30,977 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:22:31,618 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:32,618 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:34,618 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:36,619 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:38,620 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:46,119 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:22:46,119 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:22:46,652 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:22:49,957 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:22:49,958 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:22:49,958 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:22:50,625 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:22:51,625 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:52,626 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:53,626 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:55,627 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:57,628 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:22:59,629 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:01,252 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:23:01,252 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:23:09,995 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:23:09,996 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:23:09,996 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:23:10,633 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:23:11,634 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:12,634 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:13,634 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:15,635 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:16,383 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:23:16,384 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:23:16,728 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:23:17,636 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:19,637 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:30,038 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:23:30,039 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:23:30,039 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:23:30,641 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:23:31,550 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:23:31,550 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:23:31,641 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:32,641 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:33,642 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:35,643 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:37,644 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:39,644 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:46,685 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:23:46,685 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:23:46,819 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:23:50,083 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:23:50,083 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:23:50,084 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:23:50,649 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:23:51,649 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:52,649 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:53,650 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:55,651 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:57,651 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:23:59,652 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:01,821 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:24:01,821 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:24:10,128 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:24:10,128 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:24:10,129 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:24:10,657 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:24:11,657 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:12,657 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:13,658 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:15,659 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:16,883 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:24:16,956 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:24:16,956 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:24:17,660 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:19,661 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:30,165 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:24:30,166 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:24:30,166 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:24:30,665 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:24:31,665 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:32,118 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:24:32,119 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:24:32,666 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:33,666 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:35,667 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:37,668 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:39,669 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:46,949 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:24:47,250 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:24:47,250 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:24:50,206 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:24:50,206 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:24:50,207 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:24:50,673 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:24:51,674 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:52,674 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:53,674 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:55,675 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:57,676 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:24:59,677 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:02,380 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:25:02,380 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:25:10,249 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:25:10,249 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:25:10,250 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:25:10,682 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:25:11,682 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:12,682 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:13,683 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:15,684 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:17,017 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:25:17,514 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:25:17,515 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:25:17,684 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:19,685 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:30,290 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:25:30,290 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:25:30,291 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:25:30,690 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:25:31,690 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:32,654 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:25:32,654 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:25:32,690 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:33,691 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:35,692 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:37,693 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:39,694 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:47,090 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:25:47,798 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:25:47,798 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:25:50,327 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:25:50,327 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:25:50,328 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:25:50,699 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:25:51,699 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:52,700 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:53,700 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:55,701 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:57,702 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:25:59,703 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:02,932 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:26:02,932 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:26:10,363 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:26:10,363 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:26:10,364 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:26:10,708 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:26:11,708 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:12,708 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:13,709 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:15,710 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:17,163 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:26:17,710 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:18,063 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:26:18,063 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:26:19,711 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:30,405 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:26:30,405 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:26:30,406 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:26:30,716 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:26:31,716 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:32,717 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:33,208 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:26:33,208 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:26:33,717 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:35,718 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:37,719 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:39,720 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:47,244 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:26:48,339 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:26:48,339 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:26:50,443 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:26:50,443 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:26:50,444 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:26:50,724 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:26:51,725 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:52,725 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:53,726 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:55,727 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:57,727 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:26:59,728 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:03,470 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:27:03,470 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:27:10,487 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:27:10,487 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:27:10,487 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:27:10,733 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:27:11,733 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:12,734 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:13,734 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:15,735 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:17,325 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:27:17,736 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:18,605 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:27:18,605 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:27:19,737 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:30,522 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:27:30,523 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:27:30,523 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:27:30,741 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:27:31,741 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:32,741 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:33,742 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:33,837 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:27:33,837 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:27:35,743 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:37,744 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:39,744 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:47,400 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:27:48,968 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:27:48,968 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:27:50,561 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:27:50,561 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:27:50,562 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:27:50,749 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:27:51,749 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:52,750 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:53,750 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:55,751 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:57,752 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:27:59,753 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:04,099 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:28:04,100 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:28:10,599 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:28:10,599 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:28:10,600 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:28:10,757 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:28:11,757 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:12,758 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:13,758 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:15,759 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:17,476 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:28:17,760 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:19,232 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:28:19,232 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:28:19,761 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:30,634 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:28:30,634 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:28:30,635 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:28:30,765 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:28:31,766 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:32,766 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:33,767 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:34,370 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:28:34,370 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:28:35,767 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:37,768 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:39,769 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:47,553 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:28:49,502 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:28:49,502 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:28:50,675 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:28:50,675 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:28:50,676 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:28:50,774 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:28:51,774 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:52,775 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:53,775 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:55,776 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:57,776 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:28:59,777 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:04,632 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:29:04,633 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:29:10,711 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:29:10,711 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:29:10,712 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:29:10,781 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:29:11,782 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:12,782 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:13,783 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:15,783 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:17,628 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:29:17,784 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:19,766 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:29:19,767 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:29:19,785 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:30,748 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:29:30,749 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:29:30,749 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:29:30,790 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:29:31,790 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:32,790 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:33,791 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:34,898 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:29:34,898 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:29:35,791 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:37,792 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:39,793 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:47,705 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:29:50,030 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:29:50,030 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:29:50,785 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:29:50,785 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:29:50,786 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:29:50,797 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:29:51,797 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:52,798 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:53,798 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:55,799 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:57,800 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:29:59,801 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:05,201 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:30:05,202 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:30:10,821 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:30:10,822 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:30:10,822 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:30:11,805 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:30:11,806 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:13,806 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:15,807 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:17,778 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:30:17,808 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:19,809 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:20,340 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:30:20,341 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:30:30,857 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:30:30,858 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:30:30,858 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:30:31,814 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:30:31,814 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:33,815 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:35,471 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:30:35,472 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:30:35,816 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:37,817 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:39,818 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:47,857 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:30:50,603 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:30:50,603 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:30:50,895 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:30:50,895 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:30:50,895 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:30:51,822 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:30:51,823 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:53,823 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:55,824 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:57,825 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:30:59,826 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:05,735 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:31:05,735 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:31:10,930 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:31:10,930 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:31:10,931 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:31:11,831 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:31:11,831 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:13,831 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:15,832 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:17,833 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:17,934 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:31:19,834 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:20,869 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:31:20,869 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:31:30,971 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:31:30,971 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:31:30,972 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:31:31,839 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:31:31,839 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:33,840 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:35,841 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:36,002 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:31:36,002 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:31:37,842 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:39,842 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:48,013 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:31:51,011 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:31:51,012 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:31:51,012 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:31:51,142 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:31:51,142 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:31:51,847 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:31:51,848 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:53,848 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:55,849 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:57,850 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:31:59,851 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:06,410 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:32:06,410 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:32:11,053 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:32:11,053 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:32:11,053 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:32:11,856 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:32:11,856 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:13,857 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:15,857 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:17,858 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:18,093 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:32:19,859 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:21,550 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:32:21,550 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:32:31,094 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:32:31,094 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:32:31,094 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:32:31,864 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:32:31,864 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:33,865 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:35,865 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:36,680 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:32:36,680 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:32:38,867 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:40,868 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:48,172 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:32:51,129 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:32:51,130 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:32:51,131 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:32:51,818 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:32:51,818 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:32:51,872 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:32:52,873 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:53,873 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:54,873 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:56,874 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:32:58,875 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:00,876 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:06,981 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:33:06,982 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:33:11,168 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:33:11,168 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:33:11,169 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:33:11,880 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:33:12,881 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:13,881 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:14,882 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:16,882 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:18,250 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:33:18,883 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:20,884 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:22,114 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:33:22,114 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:33:31,207 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:33:31,207 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:33:31,208 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:33:31,889 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:33:32,889 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:33,889 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:34,890 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:36,891 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:37,244 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:33:37,245 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:33:38,892 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:40,892 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:48,327 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:33:51,247 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:33:51,247 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:33:51,247 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:33:51,897 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:33:51,897 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:52,392 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:33:52,392 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:33:52,897 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:54,898 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:56,899 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:33:58,900 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:00,900 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:07,532 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:34:07,533 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:34:11,286 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:34:11,287 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:34:11,287 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:34:11,905 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:34:11,905 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:12,905 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:14,906 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:16,907 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:18,403 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:34:18,908 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:20,909 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:22,663 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:34:22,664 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:34:31,321 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:34:31,321 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:34:31,322 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:34:31,913 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:34:31,913 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:32,913 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:34,914 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:36,915 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:37,804 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:34:37,805 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:34:38,916 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:40,917 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:48,482 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:34:51,363 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:34:51,364 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:34:51,364 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:34:51,921 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:34:52,922 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:52,940 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:34:52,940 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:34:53,922 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:54,922 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:56,923 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:34:58,924 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:00,925 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:08,105 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:35:08,105 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:35:11,399 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:35:11,399 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:35:11,400 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:35:11,930 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:35:12,930 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:13,930 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:14,931 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:16,932 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:18,560 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:35:18,933 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:20,933 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:23,254 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:35:23,254 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:35:31,434 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:35:31,434 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:35:31,435 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:35:31,938 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:35:32,938 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:33,939 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:34,939 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:36,940 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:38,387 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:35:38,388 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:35:38,941 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:40,942 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:48,638 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:35:51,468 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:35:51,468 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:35:51,469 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:35:51,946 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:35:52,947 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:53,531 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:35:53,531 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:35:53,947 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:54,948 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:56,949 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:35:58,949 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:00,950 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:08,692 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:36:08,692 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:36:11,504 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:36:11,504 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:36:11,504 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:36:11,955 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:36:12,955 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:13,956 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:14,956 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:16,957 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:18,716 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:36:18,958 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:20,959 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:23,829 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:36:23,829 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:36:31,539 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:36:31,540 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:36:31,540 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:36:31,963 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:36:32,963 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:33,964 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:34,964 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:36,965 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:38,965 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:36:38,965 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:36:38,968 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:40,969 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:48,793 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:36:51,574 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:36:51,575 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:36:51,575 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:36:51,973 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:36:52,974 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:53,974 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:54,110 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:36:54,110 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:36:54,975 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:56,976 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:36:58,976 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:00,977 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:09,251 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:37:09,251 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:37:11,609 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:37:11,609 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:37:11,610 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:37:11,982 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:37:11,982 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:12,982 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:14,983 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:16,984 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:18,870 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:37:18,985 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:20,985 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:24,383 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:37:24,384 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:37:31,648 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:37:31,649 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:37:31,649 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:37:31,990 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:37:31,990 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:32,990 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:34,991 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:36,992 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:38,993 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:39,518 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:37:39,518 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:37:40,994 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:48,948 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:37:51,686 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:37:51,686 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:37:51,687 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:37:51,998 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:37:51,998 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:52,999 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:54,651 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:37:54,652 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:37:54,999 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:57,000 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:37:59,001 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:01,002 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:09,787 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:38:09,787 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:38:11,723 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:38:11,724 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:38:11,724 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:38:12,006 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:38:12,006 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:13,006 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:15,007 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:17,008 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:19,009 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:19,024 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:38:21,010 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:24,919 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:38:24,919 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:38:31,762 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:38:31,763 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:38:31,763 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:38:32,014 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:38:33,014 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:35,015 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:37,016 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:39,017 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:40,049 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:38:40,049 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:38:41,018 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:49,100 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:38:51,801 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:38:51,801 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:38:51,802 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:38:52,022 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:38:53,023 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:55,023 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:55,181 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:38:55,182 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:38:57,024 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:38:59,025 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:01,026 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:10,313 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:39:10,313 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:39:11,839 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:39:11,840 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:39:11,840 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:39:12,030 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:39:13,030 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:15,031 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:17,032 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:19,033 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:19,176 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:39:21,034 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:25,442 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:39:25,443 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:39:31,875 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:39:31,876 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:39:31,876 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:39:32,038 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:39:33,038 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:35,039 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:37,040 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:39,041 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:40,572 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:39:40,573 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:39:41,042 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:49,253 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:39:51,911 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:39:51,912 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:39:51,912 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:39:52,046 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:39:53,047 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:55,048 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:55,703 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:39:55,703 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:39:57,048 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:39:59,049 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:10,833 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:40:10,833 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:40:11,949 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:40:11,950 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:40:11,950 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:40:12,054 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:40:13,055 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:15,055 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:17,056 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:19,057 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:19,331 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:40:21,058 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:25,966 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:40:25,966 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:40:31,992 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:40:31,993 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:40:31,993 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:40:32,062 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:40:33,062 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:35,063 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:37,064 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:39,065 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:41,065 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:41,095 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:40:41,095 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:40:49,408 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:40:52,032 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:40:52,032 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:40:52,033 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:40:52,070 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:40:53,070 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:55,071 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:56,226 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:40:56,226 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:40:57,072 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:40:59,073 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:01,073 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:11,357 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:41:11,357 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:41:12,083 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:41:12,084 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:41:12,084 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:41:13,078 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:41:13,078 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:15,079 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:17,079 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:19,080 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:19,487 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:41:21,081 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:26,488 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:41:26,489 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:41:32,138 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:41:32,138 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:41:32,139 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:41:33,086 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:41:33,086 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:35,087 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:37,087 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:39,088 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:41,089 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:41,622 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:41:41,623 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:41:49,565 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:41:52,174 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:41:52,174 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:41:52,175 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:41:53,093 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:41:53,093 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:55,094 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:56,759 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:41:56,759 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:41:57,095 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:41:59,095 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:01,096 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:11,891 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:42:11,891 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:42:12,212 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:42:12,212 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:42:12,212 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:42:13,101 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:42:13,101 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:14,101 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:16,102 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:18,103 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:19,640 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:42:20,103 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:22,104 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:24,105 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:26,106 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:27,022 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:42:27,023 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:42:28,106 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:30,107 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:32,108 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:34,109 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:36,109 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:38,110 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:40,111 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:42,112 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:42,155 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:42:42,155 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:42:44,112 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:46,113 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:48,114 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:49,714 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:42:50,114 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:52,115 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:54,116 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:56,117 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:42:57,310 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:42:57,310 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:42:58,118 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:00,119 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:02,120 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:04,120 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:06,121 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:08,122 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:10,123 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:12,124 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:12,442 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:43:12,442 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:43:14,125 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:16,060 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:43:16,060 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:43:16,061 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:43:16,126 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:43:18,127 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:19,788 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:43:20,128 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:22,129 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:24,129 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:27,572 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:43:27,573 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:43:36,118 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:43:36,118 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:43:36,119 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:43:36,134 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:43:37,135 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:38,135 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:40,136 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:42,137 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:42,704 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:43:42,704 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:43:44,138 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:49,859 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:43:56,175 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:43:56,176 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:43:56,176 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:43:57,143 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:43:57,143 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:43:57,875 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:43:57,876 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:43:58,144 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:00,144 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:02,145 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:04,146 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:13,020 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:44:13,020 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:44:16,217 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:44:16,217 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:44:16,218 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:44:17,152 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:44:17,153 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:18,153 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:19,936 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:44:20,154 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:22,154 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:24,155 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:28,152 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:44:28,152 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:44:36,254 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:44:36,254 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:44:36,255 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:44:37,161 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:44:37,161 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:38,161 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:40,162 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:42,163 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:43,286 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:44:43,286 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:44:44,164 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:50,012 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:44:56,290 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:44:56,290 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:44:56,291 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:44:57,169 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:44:57,169 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:58,169 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:44:58,431 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:44:58,431 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:45:00,170 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:02,171 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:04,172 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:13,562 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:45:13,562 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:45:16,326 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:45:16,326 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:45:16,327 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:45:17,177 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:45:17,177 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:18,177 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:20,090 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:45:20,178 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:22,178 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:24,179 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:28,694 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:45:28,694 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:45:36,363 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:45:36,363 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:45:36,364 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:45:37,185 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:45:37,185 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:38,185 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:40,186 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:42,187 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:43,826 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:45:43,826 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:45:44,187 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:50,167 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:45:56,399 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:45:56,399 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:45:56,400 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:45:57,192 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:45:57,192 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:58,192 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:45:58,959 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:45:58,959 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:46:00,193 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:02,194 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:04,195 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:14,094 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:46:14,095 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:46:16,435 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:46:16,435 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:46:16,436 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:46:17,200 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:46:17,200 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:19,201 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:20,241 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:46:21,202 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:23,203 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:25,204 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:29,225 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:46:29,225 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:46:36,477 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:46:36,477 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:46:36,477 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:46:37,209 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:46:38,209 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:39,209 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:41,210 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:43,211 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:44,355 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:46:44,355 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:46:45,212 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:50,317 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:46:56,514 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:46:56,514 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:46:56,514 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:46:57,217 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:46:58,217 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:59,218 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:46:59,488 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:46:59,488 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:47:01,219 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:03,220 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:05,221 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:14,622 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:47:14,622 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:47:16,553 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:47:16,553 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:47:16,554 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:47:17,225 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:47:18,226 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:19,226 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:20,393 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:47:21,227 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:23,228 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:25,229 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:29,754 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:47:29,755 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:47:36,589 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:47:36,589 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:47:36,589 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:47:37,234 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:47:38,234 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:39,235 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:41,235 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:43,236 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:44,885 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:47:44,886 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:47:45,237 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:50,470 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:47:56,623 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:47:56,624 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:47:56,624 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:47:57,243 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:47:58,243 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:47:59,243 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:00,015 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:48:00,016 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:48:01,244 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:03,245 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:05,246 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:15,147 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:48:15,147 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:48:16,661 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:48:16,661 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:48:16,662 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:48:17,251 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:48:18,251 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:19,251 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:20,546 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:48:21,252 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:23,253 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:25,254 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:30,278 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:48:30,278 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:48:36,700 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:48:36,700 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:48:36,701 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:48:37,259 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:48:38,259 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:39,260 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:41,260 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:43,261 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:45,262 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:45,411 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:48:45,411 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:48:50,623 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:48:56,738 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:48:56,739 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:48:56,739 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:48:57,267 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:48:58,268 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:48:59,268 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:00,544 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:49:00,545 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:49:01,269 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:03,270 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:05,271 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:15,681 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:49:15,682 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:49:16,779 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:49:16,779 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:49:16,780 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:49:17,276 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:49:18,276 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:19,277 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:20,701 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:49:21,277 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:23,279 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:25,279 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:30,812 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:49:30,812 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:49:36,819 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:49:36,819 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:49:36,819 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:49:37,284 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:49:38,284 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:39,285 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:41,286 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:43,287 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:45,287 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:45,947 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:49:45,947 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:49:50,779 DEBUG SenderThread:708348 [sender.py:send():179] send: stats +2021-07-15 16:49:56,860 DEBUG SenderThread:708348 [sender.py:send():179] send: history +2021-07-15 16:49:56,861 DEBUG SenderThread:708348 [sender.py:send():179] send: summary +2021-07-15 16:49:56,861 INFO SenderThread:708348 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 16:49:57,292 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/wandb-summary.json +2021-07-15 16:49:58,293 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:49:59,293 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:50:01,467 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:50:01,468 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:50:04,296 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:50:07,297 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:50:09,298 INFO Thread-8 :708348 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_152938-8qznp93p/files/output.log +2021-07-15 16:50:16,636 DEBUG HandlerThread:708348 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 16:50:16,637 DEBUG SenderThread:708348 [sender.py:send_request():193] send_request: stop_status +2021-07-15 16:50:20,859 DEBUG SenderThread:708348 [sender.py:send():179] send: stats