diff --git "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" --- "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" +++ "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" @@ -5235,3 +5235,1460 @@ 2021-07-15 21:28:31,455 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status 2021-07-15 21:28:31,456 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status 2021-07-15 21:28:39,389 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:28:46,586 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:28:46,587 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:29:01,723 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:29:01,723 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:29:09,467 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:29:16,855 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:29:16,856 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:29:31,989 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:29:31,989 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:29:36,471 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:29:39,472 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:29:39,544 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:29:41,473 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:29:43,474 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:29:45,475 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:29:47,131 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:29:47,132 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:29:56,388 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:29:56,388 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:29:56,389 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:29:56,479 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:29:58,480 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:29:59,480 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:01,481 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:02,263 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:30:02,263 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:30:03,482 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:05,483 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:09,621 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:30:16,426 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:30:16,427 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:30:16,427 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:30:16,486 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:30:17,413 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:30:17,414 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:30:17,487 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:18,487 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:19,488 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:21,488 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:23,489 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:25,490 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:32,560 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:30:32,560 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:30:36,464 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:30:36,465 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:30:36,465 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:30:36,494 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:30:38,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:39,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:39,700 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:30:41,496 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:43,497 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:45,497 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:47,693 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:30:47,693 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:30:56,520 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:30:56,521 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:30:56,521 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:30:57,501 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:30:58,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:30:59,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:01,503 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:02,848 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:31:02,849 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:31:03,504 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:05,505 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:09,778 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:31:16,559 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:31:16,559 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:31:16,560 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:31:17,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:31:18,023 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:31:18,023 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:31:18,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:19,510 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:21,511 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:23,512 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:25,512 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:33,166 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:31:33,166 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:31:36,595 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:31:36,595 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:31:36,596 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:31:37,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:31:37,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:38,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:39,518 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:39,856 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:31:41,518 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:43,519 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:45,520 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:48,316 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:31:48,316 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:31:56,635 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:31:56,635 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:31:56,635 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:31:57,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:31:57,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:58,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:31:59,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:01,526 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:03,445 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:32:03,445 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:32:03,527 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:05,528 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:09,935 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:32:16,689 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:32:16,689 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:32:16,689 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:32:17,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:32:17,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:18,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:18,726 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:32:18,726 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:32:19,534 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:21,535 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:23,536 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:25,536 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:33,911 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:32:33,912 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:32:36,726 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:32:36,727 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:32:36,727 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:32:37,541 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:32:37,541 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:38,542 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:39,542 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:40,015 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:32:41,543 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:43,544 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:45,545 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:49,045 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:32:49,045 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:32:56,764 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:32:56,764 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:32:56,765 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:32:57,549 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:32:58,549 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:32:59,550 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:01,550 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:03,551 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:04,184 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:33:04,184 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:33:05,552 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:10,094 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:33:16,809 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:33:16,810 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:33:16,810 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:33:17,556 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:33:17,557 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:18,557 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:19,346 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:33:19,347 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:33:19,557 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:21,558 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:23,559 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:25,559 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:34,478 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:33:34,479 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:33:36,847 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:33:36,847 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:33:36,847 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:33:37,564 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:33:37,564 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:38,564 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:39,565 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:40,171 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:33:41,565 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:43,566 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:45,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:49,613 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:33:49,613 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:33:56,892 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:33:56,892 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:33:56,893 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:33:57,572 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:33:57,572 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:58,572 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:33:59,573 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:01,574 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:03,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:04,745 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:34:04,745 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:34:05,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:10,248 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:34:16,930 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:34:16,931 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:34:16,931 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:34:17,579 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:34:17,580 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:18,580 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:19,580 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:19,878 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:34:19,878 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:34:21,581 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:23,582 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:25,583 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:35,010 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:34:35,010 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:34:36,967 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:34:36,968 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:34:36,968 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:34:37,587 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:34:37,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:38,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:39,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:40,325 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:34:41,589 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:43,590 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:45,591 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:50,145 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:34:50,145 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:34:57,024 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:34:57,024 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:34:57,025 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:34:57,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:34:57,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:58,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:34:59,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:01,598 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:03,599 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:05,285 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:35:05,285 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:35:05,599 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:10,404 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:35:17,063 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:35:17,063 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:35:17,064 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:35:17,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:35:17,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:18,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:19,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:20,418 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:35:20,419 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:35:21,606 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:23,606 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:25,607 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:35,557 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:35:35,557 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:35:37,110 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:35:37,111 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:35:37,111 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:35:37,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:35:37,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:38,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:40,481 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:35:40,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:42,613 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:44,614 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:46,615 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:35:50,691 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:35:50,691 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:35:57,156 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:35:57,157 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:35:57,157 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:35:57,619 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:35:58,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:00,621 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:02,621 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:04,622 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:05,820 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:36:05,821 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:36:06,623 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:10,559 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:36:17,193 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:36:17,193 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:36:17,194 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:36:17,627 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:36:18,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:20,629 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:20,956 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:36:20,956 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:36:22,629 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:24,630 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:26,631 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:36,092 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:36:36,092 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:36:37,230 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:36:37,230 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:36:37,230 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:36:37,635 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:36:38,635 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:40,636 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:40,638 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:36:42,637 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:44,638 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:46,639 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:36:51,223 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:36:51,223 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:36:57,266 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:36:57,267 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:36:57,267 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:36:57,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:36:58,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:00,644 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:02,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:04,646 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:06,454 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:37:06,454 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:37:06,647 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:10,715 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:37:17,303 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:37:17,303 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:37:17,304 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:37:17,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:37:18,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:19,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:20,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:21,587 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:37:21,587 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:37:22,654 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:24,655 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:26,656 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:36,725 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:37:36,725 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:37:37,340 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:37:37,340 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:37:37,341 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:37:37,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:37:38,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:39,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:40,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:40,792 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:37:42,662 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:44,663 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:46,664 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:51,858 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:37:51,858 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:37:57,375 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:37:57,375 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:37:57,376 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:37:57,668 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:37:58,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:37:59,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:00,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:02,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:04,671 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:06,672 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:06,991 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:38:06,991 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:38:10,870 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:38:17,411 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:38:17,411 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:38:17,412 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:38:17,676 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:38:18,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:19,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:20,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:22,127 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:38:22,127 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:38:22,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:24,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:26,680 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:37,258 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:38:37,259 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:38:37,474 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:38:37,474 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:38:37,475 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:38:37,684 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:38:38,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:39,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:40,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:40,948 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:38:42,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:44,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:46,688 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:52,393 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:38:52,393 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:38:57,511 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:38:57,512 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:38:57,512 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:38:57,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:38:58,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:38:59,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:00,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:02,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:04,695 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:06,696 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:07,525 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:39:07,526 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:39:11,024 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:39:17,550 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:39:17,550 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:39:17,551 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:39:17,699 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:39:18,700 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:19,700 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:20,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:22,657 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:39:22,657 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:39:22,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:24,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:26,703 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:37,584 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:39:37,584 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:39:37,585 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:39:37,707 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:39:37,826 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:39:37,826 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:39:38,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:39,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:40,709 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:41,091 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:39:42,710 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:44,710 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:46,711 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:52,983 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:39:52,983 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:39:57,622 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:39:57,623 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:39:57,623 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:39:57,715 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:39:58,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:39:59,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:00,717 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:02,717 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:04,718 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:08,114 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:40:08,115 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:40:11,161 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:40:17,677 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:40:17,677 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:40:17,678 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:40:17,723 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:40:18,723 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:19,724 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:20,724 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:22,725 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:23,253 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:40:23,253 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:40:24,726 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:26,727 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:37,722 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:40:37,722 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:40:37,723 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:40:37,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:40:38,413 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:40:38,414 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:40:38,731 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:39,732 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:40,732 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:41,234 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:40:42,733 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:44,734 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:46,735 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:53,563 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:40:53,563 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:40:57,789 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:40:57,789 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:40:57,789 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:40:58,739 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:40:58,739 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:40:59,740 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:01,740 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:03,741 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:05,742 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:07,743 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:08,704 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:41:08,704 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:41:11,312 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:41:17,854 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:41:17,854 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:41:17,855 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:41:18,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:41:19,747 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:21,748 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:23,748 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:23,835 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:41:23,835 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:41:25,749 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:27,750 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:37,900 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:41:37,900 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:41:37,901 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:41:38,754 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:41:38,988 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:41:38,988 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:41:39,755 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:41,400 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:41:41,755 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:43,756 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:45,757 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:47,758 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:41:54,131 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:41:54,131 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:41:57,938 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:41:57,938 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:41:57,939 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:41:58,763 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:41:59,764 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:01,765 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:03,766 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:05,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:09,261 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:42:09,262 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:42:11,480 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:42:17,983 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:42:17,984 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:42:17,984 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:42:18,772 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:42:19,773 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:21,774 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:23,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:24,392 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:42:24,393 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:42:25,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:38,022 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:42:38,022 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:42:38,023 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:42:38,780 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:42:39,530 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:42:39,530 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:42:39,780 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:41,558 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:42:41,781 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:43,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:45,783 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:47,783 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:42:54,669 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:42:54,670 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:42:58,060 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:42:58,060 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:42:58,061 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:42:58,787 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:42:59,787 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:01,788 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:03,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:05,790 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:07,790 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:09,802 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:43:09,803 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:43:11,635 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:43:18,101 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:43:18,101 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:43:18,105 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:43:18,795 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:43:19,795 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:21,797 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:23,797 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:24,939 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:43:24,940 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:43:25,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:27,799 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:38,146 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:43:38,146 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:43:38,147 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:43:38,804 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:43:39,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:40,702 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:43:40,703 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:43:41,715 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:43:41,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:43,806 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:45,807 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:47,808 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:43:55,835 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:43:55,835 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:43:58,182 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:43:58,182 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:43:58,183 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:43:58,812 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:43:59,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:01,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:03,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:05,816 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:07,816 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:10,976 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:44:10,976 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:44:11,793 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:44:18,221 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:44:18,221 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:44:18,222 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:44:18,821 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:44:19,821 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:21,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:23,823 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:25,824 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:26,107 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:44:26,107 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:44:27,825 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:38,262 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:44:38,263 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:44:38,263 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:44:38,829 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:44:39,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:41,237 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:44:41,237 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:44:41,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:41,872 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:44:43,831 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:45,832 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:44:56,367 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:44:56,367 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:44:58,308 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:44:58,308 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:44:58,309 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:44:58,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:44:59,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:01,838 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:03,839 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:05,840 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:07,841 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:11,499 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:45:11,500 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:45:11,949 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:45:18,346 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:45:18,346 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:45:18,347 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:45:18,846 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:45:19,846 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:21,847 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:23,847 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:25,848 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:26,630 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:45:26,630 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:45:27,849 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:38,386 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:45:38,386 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:45:38,387 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:45:38,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:45:39,854 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:40,854 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:41,765 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:45:41,766 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:45:41,855 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:42,027 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:45:43,856 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:45,857 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:47,858 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:45:56,897 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:45:56,897 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:45:58,442 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:45:58,442 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:45:58,443 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:45:58,862 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:45:59,862 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:00,863 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:01,863 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:03,864 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:05,865 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:07,866 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:12,030 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:46:12,031 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:46:12,159 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:46:18,481 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:46:18,481 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:46:18,482 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:46:18,870 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:46:19,871 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:20,871 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:21,871 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:23,872 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:25,873 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:27,161 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:46:27,161 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:46:38,519 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:46:38,519 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:46:38,520 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:46:38,878 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:46:39,878 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:40,879 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:41,879 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:42,182 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:46:42,296 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:46:42,296 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:46:43,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:45,881 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:46:57,429 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:46:57,430 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:46:58,556 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:46:58,556 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:46:58,557 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:46:58,885 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:46:59,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:00,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:01,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:03,887 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:05,888 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:12,259 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:47:12,562 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:47:12,562 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:47:18,594 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:47:18,595 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:47:18,595 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:47:18,893 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:47:19,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:20,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:21,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:23,895 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:25,896 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:27,691 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:47:27,691 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:47:38,659 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:47:38,660 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:47:38,660 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:47:38,900 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:47:39,901 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:40,901 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:41,902 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:42,338 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:47:42,821 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:47:42,822 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:47:43,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:45,904 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:47,904 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:47:57,953 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:47:57,953 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:47:58,698 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:47:58,698 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:47:58,699 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:47:58,909 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:47:59,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:00,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:01,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:03,911 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:05,912 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:07,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:12,415 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:48:13,085 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:48:13,086 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:48:18,753 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:48:18,753 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:48:18,754 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:48:18,917 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:48:19,918 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:20,918 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:21,919 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:23,920 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:25,920 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:27,921 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:28,217 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:48:28,217 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:48:38,798 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:48:38,798 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:48:38,799 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:48:38,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:48:39,926 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:40,926 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:41,927 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:42,493 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:48:43,350 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:48:43,350 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:48:43,927 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:45,928 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:48:58,488 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:48:58,489 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:48:58,834 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:48:58,835 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:48:58,835 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:48:58,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:48:59,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:00,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:01,935 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:03,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:05,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:12,571 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:49:13,623 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:49:13,623 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:49:18,890 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:49:18,890 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:49:18,891 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:49:18,941 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:49:19,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:20,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:22,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:24,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:26,945 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:28,761 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:49:28,762 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:49:28,945 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:38,951 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:49:38,951 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:49:38,952 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:49:39,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:49:40,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:42,648 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:49:42,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:43,901 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:49:43,902 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:49:44,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:46,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:48,953 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:49:58,997 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:49:58,997 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:49:58,998 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:49:59,038 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:49:59,038 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:49:59,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:50:00,958 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:02,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:04,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:06,960 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:08,961 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:12,726 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:50:14,178 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:50:14,178 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:50:19,036 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:50:19,036 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:50:19,036 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:50:19,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:50:20,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:22,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:24,967 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:26,968 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:28,969 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:29,307 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:50:29,308 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:50:39,078 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:50:39,079 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:50:39,079 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:50:39,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:50:40,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:42,802 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:50:42,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:44,438 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:50:44,439 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:50:44,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:46,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:48,976 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:50:59,124 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:50:59,124 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:50:59,125 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:50:59,593 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:50:59,593 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:50:59,979 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:51:00,979 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:02,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:04,981 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:06,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:12,871 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:51:14,724 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:51:14,724 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:51:19,161 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:51:19,161 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:51:19,162 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:51:19,986 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:51:20,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:22,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:24,988 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:26,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:29,856 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:51:29,856 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:51:39,197 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:51:39,198 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:51:39,198 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:51:39,993 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:51:40,993 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:42,944 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:51:42,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:44,995 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:45,009 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:51:45,009 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:51:46,996 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:51:59,252 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:51:59,252 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:51:59,253 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:52:00,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:52:00,148 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:52:00,148 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:52:01,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:03,002 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:05,003 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:07,004 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:09,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:13,021 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:52:15,278 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:52:15,278 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:52:19,314 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:52:19,315 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:52:19,315 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:52:20,009 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:52:21,010 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:23,011 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:25,011 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:27,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:29,013 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:30,409 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:52:30,410 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:52:39,378 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:52:39,378 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:52:39,379 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:52:40,017 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:52:41,017 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:43,018 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:43,100 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:52:45,019 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:45,545 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:52:45,545 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:52:47,019 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:49,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:52:59,415 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:52:59,416 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:52:59,416 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:53:00,024 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:53:00,705 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:53:00,705 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:53:01,024 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:03,025 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:05,026 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:07,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:09,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:13,178 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:53:15,846 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:53:15,846 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:53:19,451 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:53:19,452 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:53:19,452 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:53:20,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:53:21,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:23,032 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:25,033 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:27,034 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:29,034 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:30,979 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:53:30,979 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:53:39,511 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:53:39,512 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:53:39,512 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:53:40,038 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:53:41,039 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:43,039 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:43,255 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:53:45,040 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:46,111 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:53:46,112 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:53:47,041 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:49,042 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:53:59,556 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:53:59,556 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:53:59,556 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:54:00,045 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:54:01,046 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:01,246 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:54:01,247 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:54:03,046 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:05,047 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:07,048 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:13,333 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:54:16,377 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:54:16,378 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:54:19,608 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:54:19,609 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:54:19,609 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:54:20,053 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:54:21,053 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:22,054 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:23,054 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:25,055 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:27,056 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:31,511 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:54:31,512 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:54:39,645 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:54:39,645 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:54:39,646 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:54:40,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:54:41,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:42,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:43,061 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:43,410 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:54:45,062 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:46,648 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:54:46,649 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:54:47,062 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:54:59,684 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:54:59,684 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:54:59,685 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:55:00,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:55:01,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:01,782 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:55:01,782 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:55:02,068 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:03,068 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:05,069 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:07,069 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:09,070 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:13,489 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:55:16,919 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:55:16,920 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:55:19,722 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:55:19,723 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:55:19,723 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:55:20,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:55:21,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:22,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:23,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:25,076 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:27,077 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:29,078 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:32,051 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:55:32,051 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:55:39,760 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:55:39,760 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:55:39,761 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:55:40,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:55:41,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:42,083 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:43,083 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:43,566 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:55:45,084 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:47,085 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:55:47,183 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:55:47,184 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:55:59,802 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:55:59,803 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:55:59,803 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:56:00,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:56:01,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:02,091 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:02,327 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:56:02,327 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:56:03,091 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:05,093 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:07,093 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:13,644 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:56:17,458 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:56:17,459 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:56:19,845 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:56:19,845 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:56:19,845 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:56:20,098 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:56:21,098 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:22,099 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:23,099 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:25,100 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:27,101 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:32,589 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:56:32,589 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:56:39,887 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:56:39,887 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:56:39,888 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:56:40,105 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:56:41,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:42,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:43,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:43,721 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:56:45,107 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:47,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:47,719 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:56:47,719 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:56:49,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:56:59,923 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:56:59,923 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:56:59,923 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:57:00,112 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:57:01,112 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:02,113 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:02,849 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:57:02,849 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:57:03,113 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:05,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:07,115 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:09,115 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:13,801 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:57:17,980 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:57:17,981 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:57:19,969 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:57:19,969 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:57:19,969 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:57:20,119 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:57:22,120 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:24,121 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:26,122 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:28,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:30,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:33,126 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:57:33,127 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:57:40,002 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:57:40,003 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:57:40,003 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:57:40,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:57:42,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:43,879 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:57:44,129 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:46,130 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:48,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:57:48,255 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:57:48,256 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:57:50,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:00,044 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:58:00,045 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:58:00,045 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:58:00,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:58:02,136 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:03,386 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:58:03,387 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:58:04,137 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:06,138 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:08,139 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:13,960 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:58:18,517 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:58:18,518 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:58:20,085 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:58:20,085 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:58:20,086 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:58:20,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:58:22,144 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:24,145 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:26,146 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:28,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:33,658 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:58:33,659 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:58:40,133 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:58:40,134 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:58:40,134 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:58:40,152 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:58:42,153 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:44,038 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:58:44,154 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:46,155 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:48,155 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:58:48,791 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:58:48,792 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:59:00,196 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:59:00,197 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:59:00,197 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:59:01,160 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:59:02,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:03,924 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:59:03,924 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:59:04,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:06,162 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:08,163 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:14,115 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:59:19,056 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:59:19,056 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:59:20,240 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:59:20,240 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:59:20,240 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:59:21,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:59:22,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:24,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:26,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:28,170 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:34,185 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:59:34,186 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:59:40,303 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:59:40,303 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:59:40,304 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:59:41,174 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:59:42,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:44,176 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:44,192 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:59:46,176 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:48,177 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:59:49,320 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:59:49,321 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:59:50,178 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:00,343 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:00:00,344 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:00:00,344 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:00:01,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:00:02,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:04,183 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:04,460 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:00:04,460 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:00:06,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:08,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:10,185 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:14,270 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:00:19,596 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:00:19,597 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:00:20,398 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:00:20,399 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:00:20,399 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:00:21,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:00:22,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:24,190 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:26,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:28,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:30,192 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:34,726 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:00:34,726 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:00:40,444 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:00:40,444 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:00:40,444 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:00:41,196 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:00:42,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:44,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:44,348 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:00:46,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:48,199 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:00:49,856 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:00:49,857 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:00:50,200 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:00,489 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:01:00,490 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:01:00,490 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:01:01,204 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:01:02,204 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:04,205 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:04,987 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:01:04,987 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:01:06,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:08,207 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:14,425 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:01:20,123 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:01:20,124 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:01:20,535 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:01:20,535 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:01:20,536 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:01:21,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:01:22,212 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:24,212 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:26,213 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:28,214 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:35,257 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:01:35,257 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:01:40,580 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:01:40,581 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:01:40,581 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:01:41,219 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:01:42,219 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:44,220 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:44,497 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:01:46,221 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:48,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:01:50,386 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:01:50,386 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:02:00,625 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:02:00,625 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:02:00,626 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:02:01,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:02:02,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:04,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:05,527 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:02:05,527 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:02:06,228 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:08,229 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:10,230 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:14,575 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:02:20,664 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:02:20,664 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:02:20,792 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:02:20,792 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:02:20,793 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:02:21,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:02:22,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:24,235 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:26,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:28,237 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:30,238 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:35,800 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:02:35,800 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:02:40,709 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:02:40,710 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:02:40,710 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:02:41,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:02:42,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:44,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:44,644 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:02:46,244 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:48,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:50,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:02:50,930 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:02:50,931 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:03:00,747 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:03:00,747 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:03:00,748 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:03:01,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:03:02,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:04,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:06,066 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:03:06,066 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:03:06,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:08,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:10,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:14,721 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:03:20,784 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:03:20,784 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:03:20,785 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:03:21,220 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:03:21,220 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:03:21,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:03:22,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:24,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:26,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:28,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:30,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:36,352 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:03:36,353 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:03:40,819 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:03:40,820 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:03:40,820 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:03:41,264 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:03:42,264 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:44,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:44,799 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:03:46,266 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:48,267 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:03:51,483 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:03:51,483 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:04:00,868 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:04:00,868 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:04:00,869 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:04:01,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:04:02,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:03,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:04,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:06,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:06,618 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:04:06,618 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:04:08,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:14,877 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:04:20,904 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:04:20,905 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:04:20,905 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:04:21,279 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:04:21,774 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:04:21,774 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:04:22,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:23,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:24,281 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:26,281 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:28,282 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:36,905 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:04:36,905 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:04:40,944 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:04:40,944 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:04:40,945 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:04:41,287 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:04:42,287 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:43,288 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:44,288 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:44,954 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:04:46,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:48,290 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:04:52,044 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:04:52,044 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:05:00,983 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:05:00,984 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:05:00,984 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:05:01,295 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:05:02,295 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:03,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:04,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:06,297 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:07,174 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:05:07,174 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:05:08,298 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:10,298 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:15,032 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:05:21,025 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:05:21,025 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:05:21,026 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:05:21,302 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:05:22,302 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:22,329 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:05:22,329 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:05:23,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:24,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:26,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:28,305 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:30,305 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:37,470 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:05:37,471 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:05:41,062 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:05:41,063 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:05:41,063 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:05:41,309 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:05:42,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:43,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:44,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:45,113 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:05:46,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:48,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:50,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:05:52,604 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:05:52,605 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:06:01,100 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:06:01,101 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:06:01,101 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:06:01,316 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:06:02,316 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:03,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:04,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:06,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:07,740 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:06:07,741 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:06:08,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:10,320 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:15,189 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:06:21,138 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:06:21,139 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:06:21,139 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:06:21,323 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:06:22,324 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:22,931 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:06:22,932 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:06:23,324 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:24,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:26,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:28,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:30,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:38,080 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:06:38,081 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:06:41,181 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:06:41,181 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:06:41,182 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:06:41,331 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:06:42,331 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:43,332 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:44,332 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:45,266 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:06:46,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:48,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:06:53,211 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:06:53,212 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:07:01,244 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:07:01,244 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:07:01,245 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:07:01,338 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:07:02,339 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:03,339 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:04,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:06,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:08,341 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:08,349 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:07:08,349 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:07:15,343 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:07:21,283 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:07:21,283 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:07:21,284 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:07:21,346 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:07:22,347 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:23,347 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:23,486 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:07:23,487 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:07:25,348 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:27,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:29,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:31,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:38,644 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:07:38,644 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:07:41,323 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:07:41,324 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:07:41,324 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:07:41,354 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:07:43,354 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:45,355 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:45,420 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 22:07:47,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:49,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:51,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:07:53,783 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:07:53,784 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:08:01,362 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 22:08:01,362 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 22:08:01,363 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 22:08:02,361 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 22:08:03,361 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:08:05,362 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:08:07,363 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:08:08,912 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 22:08:08,912 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 22:08:09,363 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:08:11,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 22:08:15,500 DEBUG SenderThread:735392 [sender.py:send():179] send: stats